PageRenderTime 28ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 0ms

/core/infinit.e.processing.generic.library/src/com/ikanow/infinit/e/processing/generic/test/TestCode.java

https://github.com/IKANOW/Infinit.e
Java | 178 lines | 48 code | 18 blank | 112 comment | 0 complexity | 73858b433bd7fd26f1971221c6a755fd MD5 | raw file
Possible License(s): BSD-3-Clause
  1. /*******************************************************************************
  2. * Copyright 2012, The Infinit.e Open Source Project.
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU Affero General Public License, version 3,
  6. * as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU Affero General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU Affero General Public License
  14. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  15. ******************************************************************************/
  16. package com.ikanow.infinit.e.processing.generic.test;
  17. import java.io.IOException;
  18. import java.util.Arrays;
  19. import java.util.LinkedList;
  20. import org.bson.types.ObjectId;
  21. import com.google.gson.GsonBuilder;
  22. import com.ikanow.infinit.e.data_model.InfiniteEnums;
  23. import com.ikanow.infinit.e.data_model.store.DbManager;
  24. import com.ikanow.infinit.e.data_model.store.config.source.SourcePojo;
  25. import com.ikanow.infinit.e.data_model.store.document.AssociationPojo;
  26. import com.ikanow.infinit.e.data_model.store.document.DocumentPojo;
  27. import com.ikanow.infinit.e.data_model.Globals;
  28. import com.ikanow.infinit.e.data_model.Globals.Identity;
  29. import com.ikanow.infinit.e.harvest.HarvestController;
  30. import com.ikanow.infinit.e.processing.generic.GenericProcessingController;
  31. import com.ikanow.infinit.e.processing.generic.aggregation.AggregationManager;
  32. import com.ikanow.infinit.e.processing.generic.store_and_index.StoreAndIndexManager;
  33. import com.mongodb.BasicDBObject;
  34. @SuppressWarnings("unused")
  35. public class TestCode {
  36. /**
  37. * @param args
  38. * @throws IOException
  39. */
  40. public static void main(String[] args) throws IOException {
  41. System.out.println(Arrays.toString(args));
  42. Globals.setIdentity(Identity.IDENTITY_SERVICE);
  43. Globals.overrideConfigLocation(args[0]);
  44. // "-1"] Test initialization
  45. new GenericProcessingController().Initialize();
  46. // 0] Preparation: use the harvest library to get various sets of files to test out...
  47. HarvestController hc = new HarvestController();
  48. hc.setStandaloneMode(5);
  49. LinkedList<DocumentPojo> toAdd_feed = new LinkedList<DocumentPojo>();
  50. LinkedList<DocumentPojo> toUpdate_feed = new LinkedList<DocumentPojo>();
  51. LinkedList<DocumentPojo> toDelete_feed = new LinkedList<DocumentPojo>();
  52. // File type:
  53. BasicDBObject query = new BasicDBObject("extractType", "Feed");
  54. // A useful source known to work during V0S1 testing:
  55. query.put("key", "arstechnica.com.tech-policy.2012.10.last-android-vendor-f.147.4.");
  56. SourcePojo feedSource = SourcePojo.fromDb(DbManager.getIngest().getSource().findOne(query), SourcePojo.class);
  57. hc.harvestSource(feedSource, toAdd_feed, toUpdate_feed, toDelete_feed);
  58. System.out.println("############# Retrieved sample feed documents: " + toAdd_feed.size() + " from " + feedSource.getUrl());
  59. // 1] Test the store and index manager by itself:
  60. StoreAndIndexManager.setDiagnosticMode(true);
  61. // // Test all public calls
  62. //1.1] (resize DB)
  63. // StoreAndIndexManager storeManager = new StoreAndIndexManager();
  64. // storeManager.resizeDB(7365793L);
  65. // System.exit(0);
  66. // StoreAndIndexManager storeManager = new StoreAndIndexManager();
  67. // storeManager.addToDatastore(toAdd_feed, true);
  68. // // (addToSearch called from addToDatastore so not duplicated here)
  69. // storeManager.removeFromDatastore_byURL(toAdd_feed, true);
  70. // storeManager.removeFromDatastore_byId(toAdd_feed, true);
  71. // // (removeFromSearch called from removeFromDatastore_byId so not duplicated here)
  72. // // Finally, test the index switching/bulk indexing logic standalone:
  73. // if (toAdd_feed.size() > 4) {
  74. // toAdd_feed.get(2).setIndex("document_index");
  75. // toAdd_feed.get(3).setIndex(null);
  76. // storeManager.addToSearch(toAdd_feed);
  77. // }
  78. // if (toAdd_feed.size() > 4) {
  79. // toAdd_feed.get(2).setIndex(toAdd_feed.get(0).getIndex());
  80. // toAdd_feed.get(3).setIndex(toAdd_feed.get(0).getIndex());
  81. // toAdd_feed.get(0).setIndex(null);
  82. // toAdd_feed.get(3).setIndex(null);
  83. // storeManager.removeFromSearch(toAdd_feed);
  84. // }
  85. // if (toAdd_feed.size() < 5) {
  86. // System.out.println("************** Couldn't test StoreAndIndexManager indexing fully because <5 docs");
  87. // }
  88. // System.out.println("############# Tested StoreAndIndexManager calls");
  89. // 2] Now run through a full store/aggregation cycle
  90. // 2.0] Need to rerun the harvester because adding/removing from index changes to docs
  91. // toAdd_feed.clear(); toUpdate_feed.clear(); toDelete_feed.clear();
  92. // hc.harvestSource(feedSource, toAdd_feed, toUpdate_feed, toDelete_feed);
  93. // System.out.println("############# ReRetrieved sample feed documents");
  94. // 2.1] Logic:
  95. AggregationManager.setDiagnosticMode(true);
  96. GenericProcessingController pxControl_feed = new GenericProcessingController();
  97. // No need to do this, thoroughly tested during beta (and a bit slow)
  98. //pxControl.Initialize();
  99. // Comment in the desired test...
  100. // 2.1.1] Only adding:
  101. // (+Check that the feeds' entities statistics have been updated:)
  102. // pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  103. // System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(toAdd_feed));
  104. // 2.1.2] Updating:
  105. DocumentPojo docToUpdate1 = toAdd_feed.get(0);
  106. DocumentPojo docToUpdate2 = toAdd_feed.get(1);
  107. // Couple of options:
  108. docToUpdate1.setId(null); // (should be unnecessary, _id shouldn't have been set)
  109. docToUpdate2.setId(null);
  110. docToUpdate2.setUpdateId(new ObjectId("5277fec1256f16a6d3def633")); // (Doesn't matter what it is, just check the diagnostic from StoreAndIndexManager.removeFromSearch matches)
  111. toUpdate_feed.add(docToUpdate1);
  112. toUpdate_feed.add(docToUpdate2);
  113. pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  114. // 2.1.3] Deleting some docs, adding others
  115. // toDelete_feed.add(toAdd_feed.pop());
  116. // pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  117. // 2.1.4] More logic:
  118. // DocumentPojo doc1 = null;
  119. // for (DocumentPojo doc: toAdd_feed) {
  120. // if ((null != doc.getEntities()) && (doc.getEntities().size() > 2)) {
  121. // if ((null != doc.getAssociations()) && (doc.getAssociations().size() > 2)) {
  122. // doc1 = doc;
  123. // break;
  124. // }
  125. // }
  126. // }
  127. // if (null == doc1) {
  128. // System.out.println("!!!!!!!!!!!!!!!!! FAILED TO FIND DOC FOR AGGREGATION LOGIC TEST");
  129. // }
  130. // 2.1.4.1] Only adding, 1 new entity, 1 old entity with new alias, 1 new event, 1 old event with new verb
  131. // doc1.getEntities().get(0).setDisambiguatedName("TestingTesting123");
  132. // doc1.getEntities().get(0).setIndex("testingtesting123/person");
  133. // doc1.getEntities().get(1).setActual_name("TestingTesting123");
  134. // doc1.getAssociations().get(0).setEntity1_index("testingtesting123/person");
  135. // doc1.getAssociations().get(1).setEntity1("testingtesting123");
  136. // pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  137. // 2.1.4.2] Only adding, 1 existing entity but with a new community
  138. // doc1.getCommunityIds().add("alextest");
  139. // toAdd_feed.clear();
  140. // toAdd_feed.add(doc1);
  141. // pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  142. // 2.1.4.3a] Check that large association truncation code works
  143. // StringBuffer sb = new StringBuffer();
  144. // for (int i = 0; i < 60; ++i) sb.append("word").append(i).append(' ');
  145. // doc1.getAssociations().get(0).setEntity1(sb.toString());
  146. // doc1.getAssociations().get(1).setEntity2(sb.toString());
  147. // 2.1.4.3b] Check that large number of association fields works
  148. // AssociationPojo assoc3 = doc1.getAssociations().get(1);
  149. // for (int i = 0; i < 524; ++i) {
  150. // AssociationPojo newAssoc = AssociationPojo.fromDb(assoc3.toDb(), AssociationPojo.class); //(just a dumb clone)
  151. // newAssoc.setEntity1("test1_" + i);
  152. // newAssoc.setEntity2("test2_" + i);
  153. // doc1.getAssociations().add(newAssoc);
  154. // }
  155. // toAdd_feed.clear();
  156. // toAdd_feed.add(doc1);
  157. // pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
  158. // System.out.println("############# Tested GenericProcessingController calls, feed");
  159. }
  160. }