/core/infinit.e.processing.generic.library/src/com/ikanow/infinit/e/processing/generic/test/TestCode.java

/*******************************************************************************
 * Copyright 2012, The Infinit.e Open Source Project.
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 ******************************************************************************/
package com.ikanow.infinit.e.processing.generic.test;

import java.io.IOException;
import java.util.Arrays;
import java.util.LinkedList;

import org.bson.types.ObjectId;

import com.google.gson.GsonBuilder;
import com.ikanow.infinit.e.data_model.InfiniteEnums;
import com.ikanow.infinit.e.data_model.store.DbManager;
import com.ikanow.infinit.e.data_model.store.config.source.SourcePojo;
import com.ikanow.infinit.e.data_model.store.document.AssociationPojo;
import com.ikanow.infinit.e.data_model.store.document.DocumentPojo;
import com.ikanow.infinit.e.data_model.Globals;
import com.ikanow.infinit.e.data_model.Globals.Identity;
import com.ikanow.infinit.e.harvest.HarvestController;
import com.ikanow.infinit.e.processing.generic.GenericProcessingController;
import com.ikanow.infinit.e.processing.generic.aggregation.AggregationManager;
import com.ikanow.infinit.e.processing.generic.store_and_index.StoreAndIndexManager;
import com.mongodb.BasicDBObject;

@SuppressWarnings("unused")
public class TestCode {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {

		System.out.println(Arrays.toString(args));
		Globals.setIdentity(Identity.IDENTITY_SERVICE);
		Globals.overrideConfigLocation(args[0]);
		
		// "-1"] Test initialization
		new GenericProcessingController().Initialize();
		
		// 0] Preparation: use the harvest library to get various sets of files to test out...
		HarvestController hc = new HarvestController();
		hc.setStandaloneMode(5);
		LinkedList<DocumentPojo> toAdd_feed = new LinkedList<DocumentPojo>();
		LinkedList<DocumentPojo> toUpdate_feed = new LinkedList<DocumentPojo>();
		LinkedList<DocumentPojo> toDelete_feed = new LinkedList<DocumentPojo>();
		// File type:
		BasicDBObject query = new BasicDBObject("extractType", "Feed");
		// A useful source known to work during V0S1 testing:
		query.put("key", "arstechnica.com.tech-policy.2012.10.last-android-vendor-f.147.4.");
		
		SourcePojo feedSource = SourcePojo.fromDb(DbManager.getIngest().getSource().findOne(query), SourcePojo.class);
		hc.harvestSource(feedSource, toAdd_feed, toUpdate_feed, toDelete_feed);
		System.out.println("############# Retrieved sample feed documents: " + toAdd_feed.size() + " from " + feedSource.getUrl());
		
		// 1] Test the store and index manager by itself:
		StoreAndIndexManager.setDiagnosticMode(true);
		
//		// Test all public calls
		//1.1] (resize DB)
//		StoreAndIndexManager storeManager = new StoreAndIndexManager();
//		storeManager.resizeDB(7365793L);
//		System.exit(0);
		
//		StoreAndIndexManager storeManager = new StoreAndIndexManager();
//		storeManager.addToDatastore(toAdd_feed, true);
//		// (addToSearch called from addToDatastore so not duplicated here)
//		storeManager.removeFromDatastore_byURL(toAdd_feed, true);
//		storeManager.removeFromDatastore_byId(toAdd_feed, true);
//		// (removeFromSearch called from removeFromDatastore_byId so not duplicated here)
//				// Finally, test the index switching/bulk indexing logic standalone:
//		if (toAdd_feed.size() > 4) {
//			toAdd_feed.get(2).setIndex("document_index");
//			toAdd_feed.get(3).setIndex(null);
//			storeManager.addToSearch(toAdd_feed);
//		}
//		if (toAdd_feed.size() > 4) {
//			toAdd_feed.get(2).setIndex(toAdd_feed.get(0).getIndex());
//			toAdd_feed.get(3).setIndex(toAdd_feed.get(0).getIndex());
//			toAdd_feed.get(0).setIndex(null);
//			toAdd_feed.get(3).setIndex(null);
//			storeManager.removeFromSearch(toAdd_feed);
//		}		
//		if (toAdd_feed.size() < 5) {
//			System.out.println("************** Couldn't test StoreAndIndexManager indexing fully because <5 docs");			
//		}		
//		System.out.println("############# Tested StoreAndIndexManager calls");
			
		// 2] Now run through a full store/aggregation cycle
		
		// 2.0] Need to rerun the harvester because adding/removing from index changes to docs
//		toAdd_feed.clear(); toUpdate_feed.clear(); toDelete_feed.clear();
//		hc.harvestSource(feedSource, toAdd_feed, toUpdate_feed, toDelete_feed);
//		System.out.println("############# ReRetrieved sample feed documents");

		// 2.1] Logic:
		AggregationManager.setDiagnosticMode(true);
		GenericProcessingController pxControl_feed = new GenericProcessingController(); 
		// No need to do this, thoroughly tested during beta (and a bit slow)
		//pxControl.Initialize();
		
		// Comment in the desired test...
		
		// 2.1.1] Only adding:
		// (+Check that the feeds' entities statistics have been updated:)
//		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
//		System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(toAdd_feed));
		// 2.1.2] Updating:
		DocumentPojo docToUpdate1 = toAdd_feed.get(0);
		DocumentPojo docToUpdate2 = toAdd_feed.get(1);
		// Couple of options: 
		docToUpdate1.setId(null); // (should be unnecessary, _id shouldn't have been set)
		docToUpdate2.setId(null); 
		docToUpdate2.setUpdateId(new ObjectId("5277fec1256f16a6d3def633")); // (Doesn't matter what it is, just check the diagnostic from StoreAndIndexManager.removeFromSearch matches)
		toUpdate_feed.add(docToUpdate1);
		toUpdate_feed.add(docToUpdate2);
		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
		// 2.1.3] Deleting some docs, adding others
//		toDelete_feed.add(toAdd_feed.pop());
//		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
		// 2.1.4] More logic:
//		DocumentPojo doc1 = null;
//		for (DocumentPojo doc: toAdd_feed) { 
//			if ((null != doc.getEntities()) && (doc.getEntities().size() > 2)) {
//				if ((null != doc.getAssociations()) && (doc.getAssociations().size() > 2)) {
//					doc1 = doc;
//					break;
//				}				
//			}
//		}
//		if (null == doc1) {
//			System.out.println("!!!!!!!!!!!!!!!!! FAILED TO FIND DOC FOR AGGREGATION LOGIC TEST");
//		}
		// 2.1.4.1] Only adding, 1 new entity, 1 old entity with new alias, 1 new event, 1 old event with new verb
//		doc1.getEntities().get(0).setDisambiguatedName("TestingTesting123");
//		doc1.getEntities().get(0).setIndex("testingtesting123/person");
//		doc1.getEntities().get(1).setActual_name("TestingTesting123"); 
//		doc1.getAssociations().get(0).setEntity1_index("testingtesting123/person");
//		doc1.getAssociations().get(1).setEntity1("testingtesting123");
//		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
		// 2.1.4.2] Only adding, 1 existing entity but with a new community
//		doc1.getCommunityIds().add("alextest");
//		toAdd_feed.clear();
//		toAdd_feed.add(doc1);
//		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
		// 2.1.4.3a] Check that large association truncation code works
//		StringBuffer sb = new StringBuffer(); 
//		for (int i = 0; i < 60; ++i) sb.append("word").append(i).append(' ');
//		doc1.getAssociations().get(0).setEntity1(sb.toString());
//		doc1.getAssociations().get(1).setEntity2(sb.toString());
		// 2.1.4.3b] Check that large number of association fields works
//		AssociationPojo assoc3 = doc1.getAssociations().get(1);
//		for (int i = 0; i < 524; ++i) {
//			AssociationPojo newAssoc = AssociationPojo.fromDb(assoc3.toDb(), AssociationPojo.class); //(just a dumb clone)
//			newAssoc.setEntity1("test1_" + i);
//			newAssoc.setEntity2("test2_" + i);
//			doc1.getAssociations().add(newAssoc);
//		}
//		toAdd_feed.clear();
//		toAdd_feed.add(doc1);
//		pxControl_feed.processDocuments(InfiniteEnums.FEEDS, toAdd_feed, toUpdate_feed, toDelete_feed); // (add, update, delete)
//		System.out.println("############# Tested GenericProcessingController calls, feed");
	}

}