/tst/org/diffkit/diff/engine/tst/TestEngine.groovy

  1/**
  2 * Copyright 2010-2011 Joseph Panico
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License");
  5 * you may not use this file except in compliance with the License.
  6 * You may obtain a copy of the License at
  7 *
  8 *   http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS,
 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 * See the License for the specific language governing permissions and
 14 * limitations under the License.
 15 */
 16package org.diffkit.diff.engine.tst
 17
 18import java.io.File;
 19
 20import org.apache.commons.lang.ClassUtils;
 21
 22import groovy.util.GroovyTestCase
 23
 24import org.diffkit.common.DKComparatorChain;
 25import org.diffkit.common.DKMapKeyValueComparator;
 26import org.diffkit.db.DKDBColumn 
 27import org.diffkit.db.DKDBConnectionInfo 
 28import org.diffkit.db.DKDatabase 
 29import org.diffkit.db.DKDBFlavor 
 30import org.diffkit.db.DKDBH2Loader 
 31import org.diffkit.db.DKDBPrimaryKey 
 32import org.diffkit.db.DKDBTable;
 33import org.diffkit.db.DKDBTableDataAccess 
 34import org.diffkit.db.DKDBTableLoader 
 35import org.diffkit.diff.diffor.DKEqualsDiffor;
 36import org.diffkit.diff.engine.DKColumnComparison 
 37import org.diffkit.diff.engine.DKColumnDiff 
 38import org.diffkit.diff.engine.DKColumnModel 
 39import org.diffkit.diff.engine.DKContext 
 40import org.diffkit.diff.engine.DKDiff;
 41import org.diffkit.diff.engine.DKDiffEngine 
 42import org.diffkit.diff.engine.DKRowDiff 
 43import org.diffkit.diff.engine.DKSide;
 44import org.diffkit.diff.engine.DKSource 
 45import org.diffkit.diff.engine.DKStandardTableComparison 
 46import org.diffkit.diff.engine.DKTableModel 
 47import org.diffkit.diff.sns.DKDBSink 
 48import org.diffkit.diff.sns.DKDBSource 
 49import org.diffkit.diff.sns.DKFileSink 
 50import org.diffkit.diff.sns.DKFileSource 
 51import org.diffkit.diff.sns.DKListSource
 52import org.diffkit.diff.sns.DKListSink
 53import org.diffkit.diff.sns.DKTableModelUtil;
 54import org.diffkit.util.DKFileUtil;
 55import org.diffkit.util.DKResourceUtil;
 56import org.diffkit.util.DKStringUtil;
 57
 58
 59/**
 60 * @author jpanico
 61 */
 62public class TestEngine extends GroovyTestCase {
 63   
 64   /**
 65    * both sides use the same TableModel, but there are diffs of each sort; uses DBSources
 66    */
 67   public void testSameModelFromDBToDB(){
 68      def database = this.getDatabase()
 69      def connection = database.connection
 70      def lhsName = 'LHS2'
 71      def rhsName = 'RHS2'
 72      def lhsDBTable = this.createSimpleDBTableModel(lhsName)
 73      def rhsDBTable = this.createSimpleDBTableModel(rhsName)
 74      
 75      assert database.createTable( lhsDBTable)
 76      assert database.createTable( rhsDBTable)
 77      
 78      DKDBTableDataAccess tableDataAccess = [database]
 79      DKDBH2Loader loader = [database]
 80      
 81      lhsDBTable = tableDataAccess.getTable(lhsName.toUpperCase())
 82      rhsDBTable = tableDataAccess.getTable(rhsName.toUpperCase())
 83      assert lhsDBTable
 84      assert rhsDBTable
 85      
 86      assert this.load(lhsName, lhsDBTable, loader)
 87      assert this.load(rhsName, rhsDBTable, loader)
 88      
 89      def lhsSource = this.createDBSource(lhsDBTable, database)
 90      def rhsSource = this.createDBSource(rhsDBTable, database)
 91      def tableComparison = this.createComparison2( lhsDBTable, rhsDBTable, database)
 92      DKDBSink sink = new DKDBSink(database)
 93      DKDiffEngine engine = new DKDiffEngine()
 94      
 95      engine.diff(lhsSource, rhsSource, sink, tableComparison, null)
 96      
 97      assert lhsSource.lastIndex == 5
 98      assert rhsSource.lastIndex == 5
 99      assert database.dropTable( lhsDBTable)
100      assert database.dropTable( rhsDBTable)
101      
102      def diffContextTable = sink.diffContextTable
103      def diffTable = sink.diffTable
104      assert diffContextTable
105      assert diffTable
106      def contexts = database.readAllRows( diffContextTable)
107      assert contexts
108      assert contexts.size() == 1
109      println "context->${contexts[0]}"
110      def diffs =  database.readAllRows( diffTable)
111      assert diffs
112      assert diffs.size() == 4
113      def rowStepComparator = new DKMapKeyValueComparator("ROW_STEP")
114      def columnStepComparator = new DKMapKeyValueComparator("COLUMN_STEP")
115      def comparatorChain = new DKComparatorChain(rowStepComparator, columnStepComparator)
116      Collections.sort( diffs, comparatorChain)
117      println "diffs[0]->${diffs[0]}"
118      assert diffs[0]["CONTEXT_ID"] == contexts[0]["ID"]
119      assert diffs[0]["KIND"] == DKDiff.Kind.COLUMN_DIFF.ordinal()
120      assert diffs[0]["ROW_STEP"] == 2
121      assert diffs[0]["COLUMN_STEP"] == 1
122      assert diffs[0]["LHS"] == '1111'
123      assert diffs[0]["RHS"] == 'xxxx'
124      
125      println "diffs[3]->${diffs[3]}"
126      assert diffs[3]["CONTEXT_ID"] == contexts[0]["ID"]
127      assert diffs[3]["KIND"] == DKDiff.Kind.COLUMN_DIFF.ordinal()
128      assert diffs[3]["ROW_STEP"] == 6
129      assert diffs[3]["COLUMN_STEP"] == 1
130      assert diffs[3]["LHS"] == '6666'
131      assert diffs[3]["RHS"] == 'xxxx'
132      
133      assert database.dropTable( diffContextTable)
134      assert database.dropTable( diffTable)
135   }
136   
137   /**
138    * both sides use the same TableModel, but there are diffs of each sort; uses DBSources
139    */
140   public void testSameModelFromDB(){
141      def database = this.getDatabase()
142      def connection = database.connection
143      def lhsName = 'LHS2'
144      def rhsName = 'RHS2'
145      def lhsDBTable = this.createSimpleDBTableModel(lhsName)
146      def rhsDBTable = this.createSimpleDBTableModel(rhsName)
147      
148      assert database.createTable( lhsDBTable )
149      assert database.createTable( rhsDBTable)
150      
151      DKDBTableDataAccess tableDataAccess = [database]
152      DKDBH2Loader loader = [database]
153      
154      lhsDBTable = tableDataAccess.getTable(lhsName.toUpperCase())
155      rhsDBTable = tableDataAccess.getTable(rhsName.toUpperCase())
156      assert lhsDBTable
157      assert rhsDBTable
158      
159      assert this.load(lhsName, lhsDBTable, loader)
160      assert this.load(rhsName, rhsDBTable, loader)
161      
162      def lhsSource = this.createDBSource(lhsDBTable, database)
163      def rhsSource = this.createDBSource(rhsDBTable, database)
164      def tableComparison = this.createComparison2( lhsDBTable, rhsDBTable, database)
165      def diffFileName = 'testSameModelFromDB.diff'
166      DKFileSink sink = this.createSink2( diffFileName)
167      DKDiffEngine engine = new DKDiffEngine()
168      
169      engine.diff(lhsSource, rhsSource, sink, tableComparison, null)
170      
171      assert lhsSource.lastIndex == 5
172      assert rhsSource.lastIndex == 5
173      assert database.dropTable( lhsDBTable)
174      assert database.dropTable( rhsDBTable)
175      
176      def expectedFile = this.getExpectedFile(diffFileName)
177      def actualFile = sink.file
178      assert expectedFile
179      assert actualFile
180      
181      assert DKFileUtil.readFullyAsString(expectedFile) == DKFileUtil.readFullyAsString(actualFile)
182   }
183   
184   private File getExpectedFile(String filename_){
185      String expectedFilePath = ClassUtils.getPackageName(this.getClass()) 
186      expectedFilePath = DKStringUtil.packageNameToResourcePath(expectedFilePath) + filename_
187      return  DKResourceUtil.findResourceAsFile(expectedFilePath)
188   }
189   
190   private DKFileSink createSink2(String sinkFileName_){
191      File sinkFile = ['./'+sinkFileName_]
192      if(sinkFile.exists())
193         sinkFile.delete()
194      return new DKFileSink(sinkFile, false)
195   }
196   
197   private DKStandardTableComparison createComparison2(DKDBTable lhsDBTable_, DKDBTable rhsDBTable_, DKDatabase database_) {
198      DKTableModel lhsTableModel = DKTableModelUtil.createDefaultTableModel(database_.flavor,lhsDBTable_, null)
199      DKTableModel rhsTableModel = DKTableModelUtil.createDefaultTableModel(database_.flavor,rhsDBTable_, null)
200      
201      DKColumnComparison[] map = DKColumnComparison.createColumnPlans( lhsTableModel, rhsTableModel, (int[]) [1,2], DKEqualsDiffor.instance)
202      //    println "map->$map"
203      DKStandardTableComparison comparison = new DKStandardTableComparison(lhsTableModel, rhsTableModel, DKDiff.Kind.BOTH, map, (int[])[0,1], (int[][])[[0],[0]], (long)100)
204      //    println "comparison->$comparison"
205      return comparison
206   }
207   
208   private DKDBSource createDBSource(DKDBTable table_, DKDatabase database_) {
209      def tableModel = DKTableModelUtil.createDefaultTableModel(database.flavor,table_, null)
210      assert tableModel
211      
212      return new DKDBSource(table_.tableName, null, database_, tableModel, null, null)
213   }
214   
215   private boolean load(String name_, DKDBTable table_, DKDBTableLoader loader_){
216      def csvFile = this.getCsvFile(name_)
217      return loader_.load(table_, csvFile)
218   }
219   
220   private DKDatabase getDatabase(){
221      DKDBConnectionInfo connectionInfo = ['test', DKDBFlavor.H2,"mem:test", null, null, 'test', 'test']
222      println "connectionInfo->$connectionInfo"
223      return  new DKDatabase(connectionInfo)
224   }
225   
226   private File getCsvFile(String name_){
227      def csvFile = DKResourceUtil.findResourceAsFile(String.format('org/diffkit/diff/engine/tst/%s.csv',name_))
228      println "csvFile->$csvFile"
229      assert csvFile
230      return csvFile
231   }
232   
233   private DKDBTable createSimpleDBTableModel(String tablename_){
234      DKDBColumn column1 = ['column1', 1, 'VARCHAR', 20, true]
235      DKDBColumn column2 = ['column2', 2, 'VARCHAR', -1, true]
236      DKDBColumn column3 = ['column3', 2, 'VARCHAR', -1, true]
237      DKDBColumn[] columns = [column1, column2, column3]
238      String[] pkColNames = ['column1']
239      DKDBPrimaryKey pk = ['pk_' + tablename_, pkColNames]
240      DKDBTable table = [null, null, tablename_, columns, pk]
241      return table
242   }
243   
244   /**
245    * both sides use the same TableModel, but there are diffs of each sort; uses FileSources
246    */
247   public void testSameModelFromFile(){
248      
249      def lhsFile = this.getTestFile('lhs1.csv')
250      println "lhsFile->$lhsFile"
251      assert lhsFile
252      
253      def rhsFile = this.getTestFile('rhs1.csv')
254      println "rhsFile->$rhsFile"
255      assert rhsFile
256      
257      DKTableModel tableModel = this.createSimpleTableModel()
258      DKFileSource lhsSource = new DKFileSource(lhsFile.absolutePath,  tableModel, null, null,'\\,', true, true)
259      DKFileSource rhsSource = new DKFileSource(rhsFile.absolutePath, tableModel, null, null,'\\,',  true, true)
260      def filename = 'testSameModelFromFile.diff'
261      File sinkFile = ['./'+filename]
262      if(sinkFile.exists())
263         sinkFile.delete()
264      DKFileSink sink = [sinkFile, false]
265      DKStandardTableComparison tableComparison = this.createSimpleComparison()
266      println "tableComparison->$tableComparison"
267      DKDiffEngine engine = new DKDiffEngine()
268      
269      engine.diff(lhsSource, rhsSource, sink, tableComparison, null)
270      
271      assert sink.diffCount == 6
272      
273      String expectedFilePath = ClassUtils.getPackageName(this.getClass()) 
274      expectedFilePath = DKStringUtil.packageNameToResourcePath(expectedFilePath) + filename
275      def expectedFile = DKResourceUtil.findResourceAsFile(expectedFilePath)
276      
277      String expected = DKFileUtil.readFullyAsString( expectedFile)
278      assert expected
279      String actual = DKFileUtil.readFullyAsString( sinkFile)
280      assert actual
281      
282      assert expected == actual
283   }
284   
285   private File getTestFile(String filename_){
286      String sourceFilePath = ClassUtils.getPackageName(this.getClass()) 
287      sourceFilePath = DKStringUtil.packageNameToResourcePath(sourceFilePath) + filename_
288      return DKResourceUtil.findResourceAsFile(sourceFilePath)
289   }
290   
291   public void testOneSided(){
292      DKTableModel tableModel = this.createSimpleTableModel()
293      println "tableModel->$tableModel"
294      Object[] l1 = ['1111', '1111', 1]
295      Object[] l2 = ['1111', '1111', 2]
296      Object[] l3 = ['4444', '4444', 1]
297      Object[] l4 = ['4444', '4444', 2]
298      Object[] l5 = ['6666', '6666', 1]
299      Object[] l6 = ['6666', '6666', 2]
300      def rows= [l1,l2,l3,l4, l5, l6]
301      DKListSource lSource = [tableModel, rows]
302      println "lSource->$lSource"
303      
304      DKListSource rSource = [tableModel, []]
305      println "rSource->$rSource"
306      DKStandardTableComparison tableComparison = this.createSimpleComparison()
307      println "tableComparison->$tableComparison"
308      DKDiffEngine engine = new DKDiffEngine()
309      DKListSink sink = new DKListSink()
310      engine.diff(lSource, rSource, sink, tableComparison, null)
311      
312      assert sink.diffCount == 6
313      for(diff in sink.diffs) {
314         assert diff.kind == DKDiff.Kind.ROW_DIFF
315         assert diff.side == DKSide.LEFT
316      }
317      
318      lSource = [tableModel, []]
319      println "lSource->$lSource"
320      rSource = [tableModel, rows]
321      println "rSource->$rSource"
322      sink = new DKListSink()
323      engine.diff(lSource, rSource, sink, tableComparison, null)
324      
325      assert sink.diffCount == 6
326      for(diff in sink.diffs) {
327         assert diff.kind == DKDiff.Kind.ROW_DIFF
328         assert diff.side == DKSide.RIGHT
329      }
330   }
331   
332   /**
333    * both sides use the same TableModel, but there are diffs of each sort
334    */
335   public void testSameModelSimpleDiffs(){
336      DKTableModel tableModel = this.createSimpleTableModel()
337      println "tableModel->$tableModel"
338      Object[] l1 = ['1111', '1111', 1]
339      Object[] l2 = ['1111', '1111', 2]
340      Object[] l3 = ['4444', '4444', 1]
341      Object[] l4 = ['4444', '4444', 2]
342      Object[] l5 = ['6666', '6666', 1]
343      Object[] l6 = ['6666', '6666', 2]
344      def lRows= [l1,l2,l3,l4, l5, l6]
345      DKListSource lSource = [tableModel, lRows]
346      println "lSource->$lSource"
347      
348      Object[] r1 = ['1111', '1111', 1]
349      Object[] r2 = ['1111', 'xxxx', 2]
350      Object[] r3 = ['2222', '2222', 1]
351      Object[] r4 = ['5555', '5555', 2]
352      Object[] r5 = ['6666', 'xxxx', 1]
353      Object[] r6 = ['6666', '6666', 2]
354      def rRows= [r1,r2,r3,r4, r5, r6]
355      DKListSource rSource = [tableModel, rRows]
356      println "rSource->$rSource"
357      DKStandardTableComparison tableComparison = this.createSimpleComparison()
358      println "tableComparison->$tableComparison"
359      DKDiffEngine engine = new DKDiffEngine()
360      DKListSink sink = []
361      engine.diff(lSource, rSource, sink, tableComparison, null)
362      
363      assert sink.diffCount == 6
364      
365      assert sink.diffs[0] instanceof DKColumnDiff
366      assert sink.diffs[0].kind == DKDiff.Kind.COLUMN_DIFF
367      assert sink.diffs[0].rowStep == 2
368      assert sink.diffs[0].columnStep == 1
369      assert sink.diffs[0].rowDisplayValues == [column2:'1111:xxxx', column3:'2']
370      assert sink.diffs[0].lhs == '1111'
371      assert sink.diffs[0].rhs == 'xxxx'
372      
373      
374      assert sink.diffs[1] instanceof DKRowDiff
375      assert sink.diffs[1].kind == DKDiff.Kind.ROW_DIFF
376      assert sink.diffs[1].rowStep == 3
377      assert sink.diffs[1].side == DKSide.RIGHT
378      assert sink.diffs[1].rowKeyValues == ['2222',1]
379      assert sink.diffs[1].rowDisplayValues == [column2:'2222', column3:'1']
380      
381      assert sink.diffs[2] instanceof DKRowDiff
382      assert sink.diffs[2].kind == DKDiff.Kind.ROW_DIFF
383      assert sink.diffs[2].rowStep == 4
384      assert sink.diffs[2].side == DKSide.LEFT
385      assert sink.diffs[2].rowKeyValues == ['4444',1]
386      assert sink.diffs[2].rowDisplayValues == [column2:'4444', column3:'1']
387      
388      assert sink.diffs[3] instanceof DKRowDiff
389      assert sink.diffs[3].kind == DKDiff.Kind.ROW_DIFF
390      assert sink.diffs[3].rowStep == 5
391      assert sink.diffs[3].side == DKSide.LEFT
392      assert sink.diffs[3].rowKeyValues == ['4444',2]
393      assert sink.diffs[3].rowDisplayValues == [column2:'4444', column3:'2']
394      
395      assert sink.diffs[4] instanceof DKRowDiff
396      assert sink.diffs[4].kind == DKDiff.Kind.ROW_DIFF
397      assert sink.diffs[4].rowStep == 6
398      assert sink.diffs[4].side == DKSide.RIGHT
399      assert sink.diffs[4].rowKeyValues == ['5555',2]
400      assert sink.diffs[4].rowDisplayValues == [column2:'5555', column3:'2']
401      
402      assert sink.diffs[5] instanceof DKColumnDiff
403      assert sink.diffs[5].kind == DKDiff.Kind.COLUMN_DIFF
404      assert sink.diffs[5].rowStep == 7
405      assert sink.diffs[5].columnStep == 1
406      assert sink.diffs[5].rowDisplayValues == [column2:'6666:xxxx', column3:'1']
407      assert sink.diffs[5].lhs == '6666'
408      assert sink.diffs[5].rhs == 'xxxx'
409   }
410   
411   /**
412    * both sides are identical, so there should be no diffs
413    */
414   public void testIdenticalSources(){
415      DKDiffEngine engine = new DKDiffEngine()
416      DKSource lhSource = this.createSimpleSource()
417      DKSource rhSource = this.createSimpleSource()
418      DKStandardTableComparison tableComparison = this.createSimpleComparison()
419      DKListSink sink = []
420      println "tableComparison->$tableComparison"
421      
422      engine.diff(lhSource, rhSource, sink, tableComparison, null)
423      
424      assert sink.diffCount == 0
425   }
426   
427   public void testDiffRow(){
428      DKDiffEngine engine = new DKDiffEngine()
429      DKContext context = this.createSimpleContext()
430      println "tableComparison->$context.tableComparison.description"
431      
432      DKSource rhSource = context.rhs
433      Object[] rhRow = rhSource.getNextRow()
434      def sink = context.sink
435      
436      assert sink.getDiffCount() == 0
437      sink.open();
438      
439      engine.diffRow( rhRow, rhRow, context, sink)
440      assert sink.getDiffCount() == 0
441      
442      DKSource lhSource = context.lhs
443      lhSource.getNextRow()		
444      Object[] lhRow = lhSource.getNextRow()
445      
446      engine.diffRow( lhRow, rhRow, context, sink)
447      assert sink.getDiffCount() == 1
448      def diff = sink.diffs[0]
449      assert diff
450      assert diff instanceof DKColumnDiff
451      assert diff.kind == DKDiff.Kind.COLUMN_DIFF
452      assert diff.lhs == 'zzzz'
453      assert diff.rhs == 'aaaa'
454   }
455   
456   public void testRecordRowDiff(){
457      DKDiffEngine engine = new DKDiffEngine()
458      DKContext context = this.createSimpleContext()
459      DKSource rhSource = context.rhs
460      Object[] row = rhSource.getNextRow()
461      
462      def sink = context.sink
463      assert sink.getDiffCount() == 0
464      sink.open();
465      engine.recordRowDiff( row, DKSide.LEFT_INDEX, context, sink)
466      assert sink.getDiffCount() == 1
467      def diff = sink.diffs[0]
468      assert diff
469      assert diff instanceof DKRowDiff
470      assert diff.rowStep == 0
471      assert diff.rowKeyValues == ['zzzz', 1]
472      assert diff.rowDisplayValues == [column2:'aaaa',column3:'1']
473      assert diff.kind == DKDiff.Kind.ROW_DIFF
474      assert diff.side == DKSide.LEFT
475   }
476   
477   
478   private DKContext createSimpleContext() {
479      return new DKContext(this.createSimpleSource(), this.createSimpleSource(), new DKListSink(),this.createSimpleComparison(), null)
480   }
481   
482   private DKStandardTableComparison createSimpleComparison() {
483      DKTableModel tableModel = this.createSimpleTableModel();
484      DKColumnComparison[] map = DKColumnComparison.createColumnPlans( tableModel, tableModel, (int[]) [1], DKEqualsDiffor.instance)
485      //		println "map->$map"
486      DKStandardTableComparison comparison = new DKStandardTableComparison(tableModel, tableModel, DKDiff.Kind.BOTH, map, (int[])[0], (int[][])[[1,2],[1,2]], (long)100)
487      //		println "comparison->$comparison"
488      return comparison
489   }
490   
491   private DKListSource createSimpleSource(){
492      return new DKListSource(this.createSimpleTableModel(), this.createSimpleRows())
493   }
494   
495   private List<Object[]> createSimpleRows(){
496      Object[] l1 = ['zzzz', 'aaaa', 1]
497      Object[] l2 = ['zzzz', 'zzzz', 2]
498      Object[] l3 = ['bbbb', 'zzzz', 1]
499      Object[] l4 = ['aaaa', 'bbbb', 2]
500      
501      return [l1,l2,l3,l4]
502   }
503   
504   private DKTableModel createSimpleTableModel(){
505      DKColumnModel column1 = [0, 'column1', DKColumnModel.Type.STRING]
506      DKColumnModel column2 = [1, 'column2', DKColumnModel.Type.STRING]
507      DKColumnModel column3 = [2, 'column3', DKColumnModel.Type.INTEGER]
508      DKColumnModel[] columns = [column1, column2, column3]
509      int[] key = [0,2]
510      
511      return new DKTableModel("simple_table_model",columns, key)
512   }
513}