/core/infinit.e.harvest.library/src/com/ikanow/infinit/e/harvest/enrichment/custom/StructuredAnalysisHarvester.java

Large files files are truncated, but you can click here to view the full file

/*******************************************************************************
 * Copyright 2012, The Infinit.e Open Source Project.
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 ******************************************************************************/
package com.ikanow.infinit.e.harvest.enrichment.custom;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.script.*;

import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.ikanow.infinit.e.data_model.store.DbManager;
import com.ikanow.infinit.e.data_model.store.MongoDbUtil;
import com.ikanow.infinit.e.data_model.store.config.source.SourcePipelinePojo.DocumentSpecPojo;
import com.ikanow.infinit.e.data_model.store.config.source.SourcePojo;
import com.ikanow.infinit.e.data_model.store.config.source.StructuredAnalysisConfigPojo;
import com.ikanow.infinit.e.data_model.store.config.source.StructuredAnalysisConfigPojo.GeoSpecPojo;
import com.ikanow.infinit.e.data_model.store.config.source.StructuredAnalysisConfigPojo.EntitySpecPojo;
import com.ikanow.infinit.e.data_model.store.config.source.StructuredAnalysisConfigPojo.AssociationSpecPojo;
import com.ikanow.infinit.e.data_model.store.document.DocumentPojo;
import com.ikanow.infinit.e.data_model.store.document.EntityPojo;
import com.ikanow.infinit.e.data_model.store.document.AssociationPojo;
import com.ikanow.infinit.e.data_model.store.document.GeoPojo;
import com.ikanow.infinit.e.data_model.store.feature.geo.GeoFeaturePojo;
import com.ikanow.infinit.e.data_model.utils.GeoOntologyMapping;
import com.ikanow.infinit.e.harvest.HarvestContext;
import com.ikanow.infinit.e.harvest.HarvestController;
import com.ikanow.infinit.e.harvest.utils.DateUtility;
import com.ikanow.infinit.e.harvest.utils.AssociationUtils;
import com.ikanow.infinit.e.data_model.utils.DimensionUtility;
import com.ikanow.infinit.e.harvest.utils.HarvestExceptionUtils;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;

/**
 * StructuredAnalysisHarvester
 * @author cvitter
 */
public class StructuredAnalysisHarvester 
{
	///////////////////////////////////////////////////////////////////////////////////////////
	
	// NEW PROCESSING PIPELINE INTERFACE
	
	public void setContext(HarvestContext context) {
		_context = context;
		// Setup some globals if necessary
		if (null == _gson) {
			GsonBuilder gb = new GsonBuilder();
			_gson = gb.create();
		}
	}
	
	public void resetForNewDoc() {
		resetEntityCache();
		resetDocumentCache();
	}
	
	public void resetEntityCache() {
		// Clear geoMap before we start extracting entities and associations for each feed
		if (null != _entityMap) {
			if (!_geoMap.isEmpty()) _geoMap.clear();
			if (!_entityMap.isEmpty()) _entityMap.clear();
			// Fill in geoMap and entityMap with any existing docs/entities
			_entityMap = null;
			_geoMap = null;
		}
	}//TESTED (entity_cache_reset_test)
	
	public void resetDocumentCache() {
		this._document = null;
		this._docPojo = null;
	}
	
	// Load global functions
	// (scriptLang currently ignored)
	
	public void loadGlobalFunctions(List<String> imports, List<String> scripts, String scriptLang) 
	{
		intializeScriptEngine();
        // Pass scripts into the engine
        try {
        	// Retrieve and eval script files in s.scriptFiles
        	if (imports != null) {
        		for (String file : imports) {
        			if (null != file) {
        				_securityManager.eval(_scriptEngine, JavaScriptUtils.getJavaScriptFile(file));
        			}
        		}
        	}//(end load imports)
        	
        	// Eval script passed in s.script
        	if (null != scripts) {
        		for (String script: scripts) {
        			if (null != script) {
        				_securityManager.eval(_scriptEngine, script);
        			}
        		}
        	}//(end load scripts)        	
		} 
        catch (ScriptException e) {
			this._context.getHarvestStatus().logMessage("ScriptException: " + e.getMessage(), true);
			logger.error("ScriptException: " + e.getMessage(), e);
		}        
	}//TESTED (uah:import_and_lookup_test_uahSah.json)
	
	// Set the document level fields
	
	public void setDocumentMetadata(DocumentPojo doc, DocumentSpecPojo docMetadataConfig) throws JSONException, ScriptException {
		Gson g = _gson;
		intializeDocIfNeeded(doc, g);
		
		//TODO (INF-1938): allow setting of tags (here and in legacy code)
		
		// We'll just basically duplicate the code from executeHarvest() since it's pretty simple
		// and it isn't very easy to pull out the logic in there (which is unnecessarily complicated for
		// the pipeline version since you don't need to work out whether to generate the fields before or
		// after the other stages, you get to explicity specify)
		
		// Extract Title if applicable
		try {
			if (docMetadataConfig.title != null) {
				if (JavaScriptUtils.containsScript(docMetadataConfig.title)) {
					doc.setTitle((String)getValueFromScript(docMetadataConfig.title, null, null));
				}
				else {
					doc.setTitle(getFormattedTextFromField(docMetadataConfig.title, null));
				}
			}
		}
		catch (Exception e) {
			this._context.getHarvestStatus().logMessage("title: " + e.getMessage(), true);
			//DEBUG (don't output log messages per doc)
			//logger.error("title: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)

		// Extract display URL if applicable
		try {
			if (docMetadataConfig.displayUrl != null) {
				if (JavaScriptUtils.containsScript(docMetadataConfig.displayUrl)) {
					doc.setDisplayUrl((String)getValueFromScript(docMetadataConfig.displayUrl, null, null));
				}
				else {
					doc.setDisplayUrl(getFormattedTextFromField(docMetadataConfig.displayUrl, null));
				}
			}
		}
		catch (Exception e) {
			this._context.getHarvestStatus().logMessage("displayUrl: " + e.getMessage(), true);
			//DEBUG (don't output log messages per doc)
			//logger.error("displayUrl: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)

		// Extract Description if applicable
		try {
			if (docMetadataConfig.description != null) {
				if (JavaScriptUtils.containsScript(docMetadataConfig.description)) {
					doc.setDescription((String)getValueFromScript(docMetadataConfig.description, null, null));
				}
				else {
					doc.setDescription(getFormattedTextFromField(docMetadataConfig.description, null));
				}
			}
		}
		catch (Exception e)  {
			this._context.getHarvestStatus().logMessage("description: " + e.getMessage(), true);						
			//DEBUG (don't output log messages per doc)
			//logger.error("description: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)
		

		// Extract fullText if applicable
		try {
			if (docMetadataConfig.fullText != null) {
				if (JavaScriptUtils.containsScript(docMetadataConfig.fullText)) {
					doc.setFullText((String)getValueFromScript(docMetadataConfig.fullText, null, null));
				}
				else {
					doc.setFullText(getFormattedTextFromField(docMetadataConfig.fullText, null));
				}
			}
		}
		catch (Exception e) {
			this._context.getHarvestStatus().logMessage("fullText: " + e.getMessage(), true);
			//DEBUG (don't output log messages per doc)
			//logger.error("fullText: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)

		// Extract Published Date if applicable
		try {
			if (docMetadataConfig.publishedDate != null) {
				if (JavaScriptUtils.containsScript(docMetadataConfig.publishedDate)) {
						doc.setPublishedDate(new Date(
								DateUtility.parseDate((String)getValueFromScript(docMetadataConfig.publishedDate, null, null))));
				}
				else {
					doc.setPublishedDate(new Date(
							DateUtility.parseDate((String)getFormattedTextFromField(docMetadataConfig.publishedDate, null))));
				} 
			}
		}
		catch (Exception e) {
			this._context.getHarvestStatus().logMessage("publishedDate: " + e.getMessage(), true);
			//DEBUG (don't output log messages per doc)
			//logger.error("publishedDate: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)
		
		// Extract Document GEO if applicable
		
		try {
			if (docMetadataConfig.geotag != null) {
				doc.setDocGeo(getDocGeo(docMetadataConfig.geotag));
			}
		}
		catch (Exception e) {
			this._context.getHarvestStatus().logMessage("docGeo: " + e.getMessage(), true);						
			//DEBUG (don't output log messages per doc)
			//logger.error("docGeo: " + e.getMessage(), e);
		}
		//TESTED (fulltext_docMetaTest)
	}
	//TESTED (fulltext_docMetaTest)
	
	// Set the entities
	
	StructuredAnalysisConfigPojo _pipelineTmpConfig = null;
	
	public void setEntities(DocumentPojo doc, List<EntitySpecPojo> entSpecs) throws JSONException, ScriptException {
		intializeDocIfNeeded(doc, _gson);
		if (null == _pipelineTmpConfig) {
			_pipelineTmpConfig = new StructuredAnalysisConfigPojo();
		}
		_pipelineTmpConfig.setEntities(entSpecs);
		expandIterationLoops(_pipelineTmpConfig);
		List<EntityPojo> ents = getEntities(_pipelineTmpConfig.getEntities(), doc);
		if (null == doc.getEntities()) { // (else has already been added by getEntities)
			doc.setEntities(ents);
		}
	}
	//TESTED (both first time through, and when adding to existing entities)
	
	// Set the associations
	
	public void setAssociations(DocumentPojo doc, List<AssociationSpecPojo> assocSpecs) throws JSONException, ScriptException {
		
		//TODO (INF-1922): Allow setting of directed sentiment (here and in legacy code)
		
		intializeDocIfNeeded(doc, _gson);
		if (null == _pipelineTmpConfig) {
			_pipelineTmpConfig = new StructuredAnalysisConfigPojo();
		}
		_pipelineTmpConfig.setAssociations(assocSpecs);
		expandIterationLoops(_pipelineTmpConfig);
		List<AssociationPojo> assocs = getAssociations(_pipelineTmpConfig.getAssociations(), doc);
		if (null == doc.getAssociations()) { // (else has already been added by getAssociations)
			doc.setAssociations(assocs);
		}
	}
	//TESTED (both first time through, and when adding to existing associations)
	
	///////////////////////////////////////////////////////////////////////////////////////////
	
	// (Utility function for optimization)
	private void intializeDocIfNeeded(DocumentPojo f, Gson g) throws JSONException, ScriptException {
		if (null == _document) {
			// (don't need assocs or ents)
			List<EntityPojo> ents = f.getEntities();
			List<AssociationPojo> assocs = f.getAssociations();
			f.setEntities(null);
			f.setAssociations(null);
			try {
				
				// Convert the DocumentPojo Object to a JSON document using GsonBuilder
				String docStr = g.toJson(f);
				_document = new JSONObject(docStr);
				_docPojo = f;
				// Add the document (JSONObject) to the engine
				if (null != _scriptEngine) {
			        _scriptEngine.put("document", docStr);
			        _securityManager.eval(_scriptEngine, JavaScriptUtils.initScript);
				}
			}
			finally {
				f.setEntities(ents);
				f.setAssociations(assocs);
			}
		}
	}//TESTED
	
	///////////////////////////////////////////////////////////////////////////////////////////
	
	// Loads the caches into script
	
	public void loadLookupCaches(Map<String, ObjectId> caches, Set<ObjectId> communityIds) {
		//grab any json cache and make it available to the engine
		try
		{
			if (null != caches) {
				CacheUtils.addJSONCachesToEngine(caches, _scriptEngine, _securityManager, communityIds, _context);
			}
		}
		catch (Exception ex)
		{
			_context.getHarvestStatus().logMessage("JSONcache: " + ex.getMessage(), true);
			//(no need to log this, appears in log under source -with URL- anyway):
			//logger.error("JSONcache: " + ex.getMessage(), ex);
		}		
	}//TESTED (import_and_lookup_test_uahSah.json)
	
	///////////////////////////////////////////////////////////////////////////////////////////

	// Tidy up metadadata after processing
	
	public void removeUnwantedMetadataFields(String metaFields, DocumentPojo f)
	{
		if (null != f.getMetadata()) {
			if (null != metaFields) {
				boolean bInclude = true;
				if (metaFields.startsWith("+")) {
					metaFields = metaFields.substring(1);
				}
				else if (metaFields.startsWith("-")) {
					metaFields = metaFields.substring(1);
					bInclude = false;
				}
				String[] metaFieldArray = metaFields.split("\\s*,\\s*");
				if (bInclude) {
					Set<String> metaFieldSet = new HashSet<String>();
					metaFieldSet.addAll(Arrays.asList(metaFieldArray));
					Iterator<Entry<String,  Object[]>> metaField = f.getMetadata().entrySet().iterator();
					while (metaField.hasNext()) {
						Entry<String,  Object[]> metaFieldIt = metaField.next();
						if (!metaFieldSet.contains(metaFieldIt.getKey())) {
							metaField.remove();
						}
					}
				} 
				else { // exclude case, easier
					for (String metaField: metaFieldArray) {
						if (!metaField.contains(".")) {
							f.getMetadata().remove(metaField);
						}
						else { // more complex case, nested delete							
							MongoDbUtil.recursiveNestedMapDelete(metaField.split("\\s*\\.\\s*"), 0, f.getMetadata());						
						}
					}//(end loop over metaFields)
					
				}//(end if exclude case)
				//TESTED: include (default + explicit) and exclude cases
			}
		}//(if metadata exists)		
	}//TESTED (legacy code)
	
	public boolean rejectDoc(String rejectDocCriteria, DocumentPojo f) throws JSONException, ScriptException
	{
		return rejectDoc(rejectDocCriteria, f, true);
	}
	public boolean rejectDoc(String rejectDocCriteria, DocumentPojo f, boolean logMessage) throws JSONException, ScriptException
	{
		if (null != rejectDocCriteria) {			
			intializeDocIfNeeded(f, _gson);			
			
			Object o = getValueFromScript(rejectDocCriteria, null, null, false);
			if (null != o) {
				if (o instanceof String) {
					String rejectDoc = (String)o;
					if (null != rejectDoc) {
						if (logMessage) {
							this._context.getHarvestStatus().logMessage("SAH_reject: " + rejectDoc, true);
						}
						return true;
					}					
				}
				else if (o instanceof Boolean) {
					Boolean rejectDoc = (Boolean)o;
					if (rejectDoc) {
						if (logMessage) {
							this._context.getHarvestStatus().logMessage("SAH_reject: reason not specified", true);
						}
						return true;
					}					
				}
				else {
					if (logMessage) {
						this._context.getHarvestStatus().logMessage("SAH_reject: reason not specified", true);
					}
					return true;					
				}
			}
		}
		return false;
	}//TESTED (storageSettings_test + legacy code)
	
	public void handleDocumentUpdates(String onUpdateScript, DocumentPojo f) throws JSONException, ScriptException
	{
		// Compare the new and old docs in the case when this doc is an update
		if ((null != onUpdateScript) && (null != f.getUpdateId())) {
			// (note we must be in integrated mode - not called from source/test - if f.getId() != null)
			intializeDocIfNeeded(f, _gson);			
			
			BasicDBObject query1 = new BasicDBObject(DocumentPojo._id_, f.getUpdateId());
			BasicDBObject query2 = new BasicDBObject(DocumentPojo.updateId_, f.getUpdateId());
			BasicDBObject query = new BasicDBObject(DbManager.or_, Arrays.asList(query1, query2));

			BasicDBObject docObj = (BasicDBObject) DbManager.getDocument().getMetadata().findOne(query);
			
			if (null != docObj) {
				
				if (null == PARSING_SCRIPT) { // First time through, initialize parsing script 
					// (to convert native JS return vals into something we can write into our metadata)
					PARSING_SCRIPT = JavaScriptUtils.generateParsingScript();
				}					
				if (!_isParsingScriptInitialized) {
					_securityManager.eval(_scriptEngine, PARSING_SCRIPT);
					_isParsingScriptInitialized = true;
				}
				DocumentPojo doc = DocumentPojo.fromDb(docObj, DocumentPojo.class);
		        _scriptEngine.put("old_document", _gson.toJson(doc));
		        try {
		        	_securityManager.eval(_scriptEngine,JavaScriptUtils.initOnUpdateScript);
		        	Object returnVal = _securityManager.eval(_scriptEngine, onUpdateScript);
					BasicDBList outList = JavaScriptUtils.parseNativeJsObject(returnVal, _scriptEngine);												
					f.addToMetadata("_PERSISTENT_", outList.toArray());
		        }
		        catch (Exception e) {
		        	// Extra step here...
		        	if (null != doc.getMetadata()) { // Copy persistent metadata across...
		        		Object[] persist = doc.getMetadata().get("_PERSISTENT_");
		        		if (null != persist) {
		        			f.addToMetadata("_PERSISTENT_", persist);
		        		}						        		
						this._context.getHarvestStatus().logMessage("SAH::onUpdateScript: " + e.getMessage(), true);
						//DEBUG (don't output log messages per doc)
						//logger.error("SAH::onUpdateScript: " + e.getMessage(), e);
		        	}
		        	//(TESTED)
		        }								
				//TODO (INF-1507): need to write more efficient code to deserialize metadata?
			}
			
			_document = null;
			_docPojo = null;
			intializeDocIfNeeded(f, _gson);
			
		}//TESTED (end if callback-on-update)
	}//TESTED (legacy code)
	
	///////////////////////////////////////////////////////////////////////////////////////////
	
	// PROCESSING PIPELINE - UTILITIES
	
	// Intialize script engine - currently only Java script is supported
	
	public void intializeScriptEngine()
	{
		if (null == _scriptEngine) {
			//set up the security manager
			_securityManager = new JavascriptSecurityManager();						
			_scriptFactory = new ScriptEngineManager();
			_scriptEngine = _scriptFactory.getEngineByName("JavaScript");
			
			if (null != _unstructuredHandler) { // Also initialize the scripting engine for the UAH
				_unstructuredHandler.set_sahEngine(_scriptEngine);
				_unstructuredHandler.set_sahSecurity(_securityManager);				
			}
	        // Make the engine invocable so that we can call functions in the script
	        // using the inv.invokeFunction(function) method
	        _scriptInvoker = (Invocable) _scriptEngine;
		}//(once only)
	}//TESTED
	
	///////////////////////////////////////////////////////////////////////////////////////////
	///////////////////////////////////////////////////////////////////////////////////////////
	///////////////////////////////////////////////////////////////////////////////////////////
	
	// LEGACY CODE - USE TO SUPPORT OLD CODE FOR NOW + AS UTILITY CODE FOR THE PIPELINE LOGIC
	
	// Private class variables
	private static Logger logger;
	private JSONObject _document = null; //TODO (INF-2488): change all the JSONObject logic to LinkedHashMap and (generic) Array so can just replace this with a string...
	private DocumentPojo _docPojo = null;
	private Gson _gson = null;
	private JSONObject _iterator = null;
	private String _iteratorIndex = null;
	private static Pattern SUBSTITUTION_PATTERN = Pattern.compile("\\$([a-zA-Z._0-9]+)|\\$\\{([^}]+)\\}");
	private HashMap<String, GeoPojo> _geoMap = null;
	private HashSet<String> _entityMap = null;
	
	private HarvestContext _context;
	
	/**
	 * Default Constructor
	 */
	public StructuredAnalysisHarvester()
	{			
		logger = Logger.getLogger(StructuredAnalysisHarvester.class);
	}
	
	// Allows the unstructured handler to take advantage of text created by this
	public void addUnstructuredHandler(UnstructuredAnalysisHarvester uap) {
		_unstructuredHandler = uap;
	}
	private UnstructuredAnalysisHarvester _unstructuredHandler = null;
	
	// 
	private ScriptEngineManager _scriptFactory = null;
	private ScriptEngine _scriptEngine = null;
	private JavascriptSecurityManager _securityManager = null;
	private Invocable _scriptInvoker = null;
	private static String PARSING_SCRIPT = null;
	private boolean _isParsingScriptInitialized = false; // (needs to be done once per source)

	/**
	 * executeHarvest(SourcePojo source, List<DocumentPojo> feeds) extracts document GEO, Entities,
	 * and Associations based on the DocGeoSpec, EntitySpec, and AssociationSpec information contained
	 * within the source document's StructuredAnalysis sections
	 * @param source
	 * @param docs
	 * @return List<DocumentPojo>
	 * @throws ScriptException 
	 */
	public List<DocumentPojo> executeHarvest(HarvestController contextController, SourcePojo source, List<DocumentPojo> docs)
	{
		_context = contextController;
		if (null == _gson) {
			GsonBuilder gb = new GsonBuilder();
			_gson = gb.create();
		}
		Gson g = _gson;
		
		// Skip if the StructuredAnalysis object of the source is null 
		if (source.getStructuredAnalysisConfig() != null)
		{
			StructuredAnalysisConfigPojo s = source.getStructuredAnalysisConfig();			
			// (some pre-processing to expand the specs)
			expandIterationLoops(s);	
			
			// Instantiate a new ScriptEngineManager and create an engine to execute  
			// the type of script specified in StructuredAnalysisPojo.scriptEngine
			this.intializeScriptEngine();			
						
			this.loadLookupCaches(s.getCaches(), source.getCommunityIds());
			
			// Iterate over each doc in docs, create entity and association pojo objects
			// to add to the feed using the source entity and association spec pojos
			Iterator<DocumentPojo> it = docs.iterator();
			int nDocs = 0;
			while (it.hasNext())
			{
				DocumentPojo f = it.next();
				nDocs++;
				try 
				{ 
					resetEntityCache();
					_document = null;
					_docPojo = null;
						// (don't create this until needed, since it might need to be (re)serialized after a call
						//  to the UAH which would obviously be undesirable)
								        					
					// If the script engine has been instantiated pass the feed document and any scripts
					if (_scriptEngine != null)
					{
						List<String> scriptList = null;
						List<String> scriptFileList = null;
						try {
							// Script code embedded in source
							scriptList = Arrays.asList(s.getScript());
						}
						catch (Exception e) {}
						try {
							// scriptFiles - can contain String[] of script files to import into the engine
							scriptFileList = Arrays.asList(s.getScriptFiles());							
						}
						catch (Exception e) {}							
						this.loadGlobalFunctions(scriptFileList, scriptList, s.getScriptEngine());						
					}//TESTED
					
			// 1. Document level fields
					
					// Extract Title if applicable
					boolean bTryTitleLater = false;
					try {
						if (s.getTitle() != null)
						{
							intializeDocIfNeeded(f, g);
							if (JavaScriptUtils.containsScript(s.getTitle()))
							{
								f.setTitle((String)getValueFromScript(s.getTitle(), null, null));
							}
							else
							{
								f.setTitle(getFormattedTextFromField(s.getTitle(), null));
							}
							if (null == f.getTitle()) {
								bTryTitleLater = true;
							}
						}
					}
					catch (Exception e) 
					{
						this._context.getHarvestStatus().logMessage("title: " + e.getMessage(), true);						
						//DEBUG (don't output log messages per doc)
						//logger.error("title: " + e.getMessage(), e);
					}

					// Extract Display URL if applicable
					boolean bTryDisplayUrlLater = false;
					try {
						if (s.getDisplayUrl() != null)
						{
							intializeDocIfNeeded(f, g);
							if (JavaScriptUtils.containsScript(s.getDisplayUrl()))
							{
								f.setDisplayUrl((String)getValueFromScript(s.getDisplayUrl(), null, null));
							}
							else
							{
								f.setDisplayUrl(getFormattedTextFromField(s.getDisplayUrl(), null));
							}
							if (null == f.getDisplayUrl()) {
								bTryDisplayUrlLater = true;
							}
						}
					}
					catch (Exception e) 
					{
						this._context.getHarvestStatus().logMessage("displayUrl: " + e.getMessage(), true);						
						//DEBUG (don't output log messages per doc)
						//logger.error("displayUrl: " + e.getMessage(), e);
					}
					//TOTEST

					// Extract Description if applicable
					boolean bTryDescriptionLater = false;
					try {
						if (s.getDescription() != null)
						{
							intializeDocIfNeeded(f, g);
							if (JavaScriptUtils.containsScript(s.getDescription()))
							{
								f.setDescription((String)getValueFromScript(s.getDescription(), null, null));
							}
							else
							{
								f.setDescription(getFormattedTextFromField(s.getDescription(), null));
							}
							if (null == f.getDescription()) {
								bTryDescriptionLater = true;
							}
						}
					}
					catch (Exception e) 
					{
						this._context.getHarvestStatus().logMessage("description: " + e.getMessage(), true);						
						//DEBUG (don't output log messages per doc)
						//logger.error("description: " + e.getMessage(), e);
					}
					

					// Extract fullText if applicable
					boolean bTryFullTextLater = false;
					try {
						if (s.getFullText() != null)
						{
							intializeDocIfNeeded(f, g);
							if (JavaScriptUtils.containsScript(s.getFullText()))
							{
								f.setFullText((String)getValueFromScript(s.getFullText(), null, null));
							}
							else
							{
								f.setFullText(getFormattedTextFromField(s.getFullText(), null));
							}
							if (null == f.getFullText()) {
								bTryFullTextLater = true;
							}
						}
					}
					catch (Exception e) 
					{
						this._context.getHarvestStatus().logMessage("fullText: " + e.getMessage(), true);						
						//DEBUG (don't output log messages per doc)
						//logger.error("fullText: " + e.getMessage(), e);
					}
	
					// Published date is done after the UAH 
					// (since the UAH can't access it, and it might be populated via the UAH)
					
			// 2. UAH/extraction properties
					
					// Add fields to metadata that can be used to create entities and associations
					// (Either with the UAH, or with the entity extractor)
					try {
						boolean bMetadataChanged = false;
						if (null != this._unstructuredHandler) 
						{
							try 
							{
								this._unstructuredHandler.set_sahEngine(_scriptEngine);
								this._unstructuredHandler.set_sahSecurity(_securityManager);
								bMetadataChanged = this._unstructuredHandler.executeHarvest(_context, source, f, (1 == nDocs), it.hasNext());
							}
							catch (Exception e) {
								contextController.handleExtractError(e, source); //handle extractor error if need be		
								
								it.remove(); // remove the document from the list...
								f.setTempSource(null); // (can safely corrupt this doc since it's been removed)
								
								// (Note: this can't be source level error, so carry on harvesting - unlike below)
								continue;
							}
						}	
						if (contextController.isEntityExtractionRequired(source))
						{
							bMetadataChanged = true;
							
							// Text/Entity Extraction 
							List<DocumentPojo> toAdd = new ArrayList<DocumentPojo>(1);
							toAdd.add(f);
							try {
								contextController.extractTextAndEntities(toAdd, source, false, false);
								if (toAdd.isEmpty()) { // this failed... 
									it.remove(); // remove the document from the list...
									f.setTempSource(null); // (can safely corrupt this doc since it's been removed)
									continue;
								}//TESTED
							}
							catch (Exception e) {
								contextController.handleExtractError(e, source); //handle extractor error if need be				
								it.remove(); // remove the document from the list...
								f.setTempSource(null); // (can safely corrupt this doc since it's been removed)
								
								if (source.isHarvestBadSource())
								{
									// Source error, ignore all other documents
									while (it.hasNext()) {
										f = it.next();
										f.setTempSource(null); // (can safely corrupt this doc since it's been removed)
										it.remove();
									}
									break;
								}
								else {
									continue;
								}
								//TESTED
							}
						}
						if (bMetadataChanged) {
							// Ugly, but need to re-create doc json because metadata has changed
							String sTmpFullText = f.getFullText();
							f.setFullText(null); // (no need to serialize this, can save some cycles)
							_document = null;
							_docPojo = null;
							intializeDocIfNeeded(f, g);							
					        f.setFullText(sTmpFullText); //(restore)
						}
						
						// Can copy metadata from old documents to new ones:						
						handleDocumentUpdates(s.getOnUpdateScript(), f);
						
						// Check (based on the metadata and entities so far) whether to retain the doc
						if (rejectDoc(s.getRejectDocCriteria(), f)) {
							it.remove(); // remove the document from the list...
							f.setTempSource(null); // (can safely corrupt this doc since it's been removed)
							continue;															
						}
					}
					catch (Exception e) {
						this._context.getHarvestStatus().logMessage("SAH->UAH: " + e.getMessage(), true);						
						//DEBUG (don't output log messages per doc)
						//logger.error("SAH->UAH: " + e.getMessage(), e);
					}
						
					// Now create document since there's no risk of having to re-serialize
					intializeDocIfNeeded(f, g);
					
			// 3. final doc-level metadata fields:
					
					// If description was null before might need to get it from a UAH field
					if (bTryTitleLater) {
						try {
							if (s.getTitle() != null)
							{
								intializeDocIfNeeded(f, g);
								if (JavaScriptUtils.containsScript(s.getTitle()))
								{
									f.setTitle((String)getValueFromScript(s.getTitle(), null, null));
								}
								else
								{
									f.setTitle(getFormattedTextFromField(s.getTitle(), null));
								}
							}
						}
						catch (Exception e) 
						{
							this._context.getHarvestStatus().logMessage("title: " + e.getMessage(), true);						
							//DEBUG (don't output log messages per doc)
							//logger.error("title: " + e.getMessage(), e);
						}
					}
					
					// Extract Display URL if needed
					if (bTryDisplayUrlLater) {
						try {
							if (s.getDisplayUrl() != null)
							{
								intializeDocIfNeeded(f, g);
								if (JavaScriptUtils.containsScript(s.getDisplayUrl()))
								{
									f.setDisplayUrl((String)getValueFromScript(s.getDisplayUrl(), null, null));
								}
								else
								{
									f.setDisplayUrl(getFormattedTextFromField(s.getDisplayUrl(), null));
								}
							}
						}
						catch (Exception e) 
						{
							this._context.getHarvestStatus().logMessage("displayUrl: " + e.getMessage(), true);						
							//DEBUG (don't output log messages per doc)
							//logger.error("displayUrl: " + e.getMessage(), e);
						}
					}					
					//TOTEST
					
					// If description was null before might need to get it from a UAH field
					if (bTryDescriptionLater) {
						try {
							if (s.getDescription() != null)
							{
								intializeDocIfNeeded(f, g);
								if (JavaScriptUtils.containsScript(s.getDescription()))
								{
									f.setDescription((String)getValueFromScript(s.getDescription(), null, null));
								}
								else
								{
									f.setDescription(getFormattedTextFromField(s.getDescription(), null));
								}
							}
						}
						catch (Exception e) 
						{
							this._context.getHarvestStatus().logMessage("description2: " + e.getMessage(), true);						
							//DEBUG (don't output log messages per doc)
							//logger.error("description2: " + e.getMessage(), e);
						}						
					}
					
					// If fullText was null before might need to get it from a UAH field
					if (bTryFullTextLater) {
						try {
							if (s.getFullText() != null)
							{
								intializeDocIfNeeded(f, g);
								if (JavaScriptUtils.containsScript(s.getFullText()))
								{
									f.setFullText((String)getValueFromScript(s.getFullText(), null, null));
								}
								else
								{
									f.setFullText(getFormattedTextFromField(s.getFullText(), null));
								}
							}
						}
						catch (Exception e) 
						{
							this._context.getHarvestStatus().logMessage("fullText2: " + e.getMessage(), true);						
							//DEBUG (don't output log messages per doc)
							//logger.error("fullText2: " + e.getMessage(), e);
						}						
					}
					
					// Extract Published Date if applicable
					if (s.getPublishedDate() != null)
					{
						if (JavaScriptUtils.containsScript(s.getPublishedDate()))
						{
							try 
							{
								f.setPublishedDate(new Date(
										DateUtility.parseDate((String)getValueFromScript(s.getPublishedDate(), null, null))));
							}
							catch (Exception e) 
							{
								this._context.getHarvestStatus().logMessage("publishedDate: " + e.getMessage(), true);						
							}
						}
						else
						{
							try 
							{ 
								f.setPublishedDate(new Date(
										DateUtility.parseDate((String)getFormattedTextFromField(s.getPublishedDate(), null))));
							} 
							catch (Exception e) 
							{
								this._context.getHarvestStatus().logMessage("publishedDate: " + e.getMessage(), true);						
							}
						}
					}
					
			// 4. Entity level fields		
					
					// Extract Document GEO if applicable
					
					if (s.getDocumentGeo() != null)
					{
						try
						{
							f.setDocGeo(getDocGeo(s.getDocumentGeo()));
						}
						catch (Exception e)
						{
							this._context.getHarvestStatus().logMessage("docGeo: " + e.getMessage(), true);						
						}
					}

					// Extract Entities
					if (s.getEntities() != null)
					{
						f.setEntities(getEntities(s.getEntities(), f));
					}

					// Extract Associations
					if (s.getAssociations() != null)
					{
						f.setAssociations(getAssociations(s.getAssociations(), f));
					}
					
			// 5. Remove unwanted metadata fields
					
					removeUnwantedMetadataFields(s.getMetadataFields(), f);					
				} 
				catch (Exception e) 
				{
					this._context.getHarvestStatus().logMessage("Unknown: " + e.getMessage(), true);						
					//DEBUG (don't output log messages per doc)
					//logger.error("Unknown: " + e.getMessage(), e);
				}
				finally
				{
					_document = null;
					_docPojo = null;
				}
			} // (end loop over documents)
		} // (end if SAH specified)	
		return docs;
	}

	/**
	 * getEntities(EntitySpecPojo e, DocumentPojo f)
	 * 
	 * @param e
	 * @param f
	 * @return List<EntityPojo>
	 * @throws JSONException 
	 */
	private List<EntityPojo> getEntities(List<EntitySpecPojo> esps, DocumentPojo f) throws JSONException
	{
		//TODO (INF-1922): should I always create in a new list and then add on? because of the entity map below...
		
		// If the feed already has entities we want to add the new entities to the list of existing entities
		List<EntityPojo> entities = null;
		if (f.getEntities() != null) 
		{ 
			entities = f.getEntities(); 
		}
		// Otherwise we create a new arraylist to hold the new entities we are adding
		else 
		{ 
			entities = new ArrayList<EntityPojo>();
		}
		repopulateEntityCacheIfNeeded(f);

		// Iterate over each EntitySpecPojo and try to create an entity, or entities, from the data
		JSONObject metadata = null;
		if (_document.has("metadata")) {
			metadata = _document.getJSONObject("metadata");
		}
		for (EntitySpecPojo esp : esps)
		{
			try {
				List<EntityPojo> tempEntities = getEntities(esp, f, metadata);
				for (EntityPojo e : tempEntities)
				{
					entities.add(e);
				}
			}
			catch (Exception e) {} // (carry on, prob just a missing field in this doc)
		}
		
		return entities;
	}
	
	
	
	/**
	 * getEntities
	 * @param esp
	 * @param f
	 * @return
	 */
	private List<EntityPojo> getEntities(EntitySpecPojo esp, DocumentPojo f, JSONObject currObj)
	{
		List<EntityPojo> entities = new ArrayList<EntityPojo>();
		
		// Does the entity contain a list of entities to iterate over - 
		if (esp.getIterateOver() != null)
		{
			try
			{
				String iterateOver = esp.getIterateOver();

				// Check to see if the arrayRoot specified exists in the current doc before proceeding
				
				Object itEl = null;
				try {
					itEl = currObj.get(iterateOver);
				}
				catch (JSONException e) {} // carry on, trapped below...
				
				if (null == itEl) {
					return entities;
				}
				JSONArray entityRecords = null;
				try {
					entityRecords = currObj.getJSONArray(iterateOver);
				}
				catch (JSONException e) {} // carry on, trapped below...
					
				if (null == entityRecords) {
					entityRecords = new JSONArray();
					entityRecords.put(itEl);
				}
				//TESTED

				// Get the type of object contained in EntityRecords[0]
				String objType = entityRecords.get(0).getClass().toString();

				/*
				 *  EntityRecords is a simple String[] array of entities
				 */
				if (objType.equalsIgnoreCase("class java.lang.String"))
				{
					// Iterate over array elements and extract entities
					for (int i = 0; i < entityRecords.length(); ++i) 
					{							
						String field = entityRecords.getString(i);
						long nIndex = Long.valueOf(i);
						
						if (null != esp.getType()) { // (else cannot be a valid entity, must just be a list)
							EntityPojo entity = getEntity(esp, field, String.valueOf(i), f);
							if (entity != null) entities.add(entity);	
						}
						
						// Does the association break out into multiple associations?
						if (esp.getEntities() != null)
						{
							// Iterate over the associations and call getAssociations recursively
							for (EntitySpecPojo subEsp : esp.getEntities())
							{	
								if (null != subEsp.getIterateOver()) {
									if (null == subEsp.getCreationCriteriaScript()) {
										_context.getHarvestStatus().logMessage(new StringBuffer("In iterator ").
												append(esp.getIterateOver()).append(", trying to loop over field '").
												append(subEsp.getIterateOver()).append("' in array of primitives.").toString(), true);
									}
									else {
										this.executeEntityAssociationValidation(subEsp.getCreationCriteriaScript(), field, Long.toString(nIndex));
									}
									// (any creation criteria script indicates user accepts it can be either) 
								}
								if (null != subEsp.getDisambiguated_name()) {
									EntityPojo entity = getEntity(subEsp, field, String.valueOf(i), f);
									if (entity != null) entities.add(entity);	
								}
							}										
						}//TESTED (error case, mixed object)
					}
				}

				/*
				 *  EntityRecords is a JSONArray
				 */
				else if (objType.equalsIgnoreCase("class org.json.JSONObject"))
				{
					// Iterate over array elements and extract entities
					for (int i = 0; i < entityRecords.length(); ++i) 
					{
						// Get JSONObject containing entity fields and pass entityElement
						// into the script engine so scripts can access it
						JSONObject savedIterator = null;
						if (_scriptEngine != null) 
						{
							_iterator = savedIterator = entityRecords.getJSONObject(i);
						}

						if (null != esp.getType()) { // (else cannot be a valid entity, must just be a list)
							EntityPojo entity = getEntity(esp, null, String.valueOf(i), f);
							if (entity != null) entities.add(entity);
						}
						
						// Does the entity break out into multiple entities?
						if (esp.getEntities() != null)
						{
							// Iterate over the entities and call getEntities recursively
							for (EntitySpecPojo subEsp : esp.getEntities())
							{	
								_iterator = savedIterator; // (reset this)
								
								List<EntityPojo> subEntities = getEntities(subEsp, f, _iterator);
								for (EntityPojo e : subEntities)
								{
									entities.add(e);
								}
							}
						}
					}
				}

				if (_iterator != currObj) { // (ie at the top level)
					_iterator = null;
				}
			}
			catch (Exception e)
			{
				//e.printStackTrace();
				//System.out.println(e.getMessage());
				//logger.error("Exception: " + e.getMessage());
			}
		}
		
		// Single entity
		else
		{
			// Does the entity break out into multiple entities?
			if (esp.getEntities() != null)
			{
				// Iterate over the entities and call getEntities recursively
				for (EntitySpecPojo subEsp : esp.getEntities())
				{	
					List<EntityPojo> subEntities = getEntities(subEsp, f, currObj);
					for (EntityPojo e : subEntities)
					{
						entities.add(e);
					}
				}
			}
			else
			{
				EntityPojo entity = getEntity(esp, null, null, f);
				if (entity != null) entities.add(entity);	
			}
		}
		
		return entities;
	}
	
	
	
	/**
	 * getEntity
	 * @param esp
	 * @param field
	 * @param index
	 * @param f
	 * @return
	 */
	private EntityPojo getEntity(EntitySpecPojo esp, String field, String index, DocumentPojo f)
	{
		// If the EntitySpecPojo or DocumentPojo is null return null
		if ((esp == null) || (f == null)) return null;
		
		try
		{
			EntityPojo e = new EntityPojo();
			
			// Parse creation criteria script to determine if the entity should be added
			if (esp.getCreationCriteriaScript() != null && JavaScriptUtils.containsScript(esp.getCreationCriteriaScript()))
			{
				boolean addEntity = executeEntityAssociationValidation(esp.getCreationCriteriaScript(), field, index);
				if (!addEntity) {
					return null;
				}
			}
			
			// Entity.disambiguous_name
			String disambiguatedName = null;
			if (JavaScriptUtils.containsScript(esp.getDisambiguated_name()))
			{
				disambiguatedName = (String)getValueFromScript(esp.getDisambiguated_name(), field, index);
			}
			else
			{
				if ((_iterator != null) && (esp.getDisambiguated_name().startsWith("$metadata.") || esp.getDisambiguated_name().startsWith("${metadata."))) {
					if (_context.isStandalone()) { // (minor message, while debugging only)
						_context.getHarvestStatus().logMessage("Warning: in disambiguated_name, using global $metadata when iterating", true);
					}
				}
				// Field - passed in via simple string array from getEntities
				if (field != null)
				{
					disambiguatedName = getFormattedTextFromField(esp.getDisambiguated_name(), field);
				}
				else
				{
					disambiguatedName = getFormattedTextFromField(esp.getDisambiguated_name(), field);
				}
			}
			
			// Only proceed if disambiguousName contains a meaningful value
			if (disambiguatedName != null && disambiguatedName.length() > 0)
			{
				e.setDisambiguatedName(disambiguatedName);
			}
			else // Always log failure to get a dname - to remove this, specify a creationCriteriaScript
			{
				_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required disambiguated_name from: ").append(esp.getDisambiguated_name()).toString(), true);
				return null;
			}
			
			// Entity.frequency (count)
			String freq = "1";
			if (esp.getFrequency() != null)
			{
				if (JavaScriptUtils.containsScript(esp.getFrequency()))
				{
					freq = getValueFromScript(esp.getFrequency(), field, index).toString();
				}
				else
				{
					freq = getFormattedTextFromField(esp.getFrequency(), field);
				}
				// Since we've specified freq, we're going to enforce it
				if (null == freq) { // failed to get it
					if (null == esp.getCreationCriteriaScript()) {
						_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required frequency from: ").append(esp.getFrequency()).toString(), true);
						return null;
					}
				}
			}

			// Try converting the freq string value to its numeric (double) representation
			Double frequency = (double) 0;
			try  
			{ 
				frequency = Double.parseDouble(freq); 
			} 
			catch (Exception e1) 
			{
				this._context.getHarvestStatus().logMessage(e1.getMessage(), true);
				return null;
			}
			
			// Only proceed if frequency > 0
			if (frequency > 0)
			{
				e.setFrequency(frequency.longValue()); // Cast to long from double
			}
			else
			{ 
				return null; 	
			}  
			
			// Entity.actual_name
			String actualName = null;
			if (esp.getActual_name() != null)
			{
				if (JavaScriptUtils.containsScript(esp.getActual_name()))
				{
					actualName = (String)getValueFromScript(esp.getActual_name(), field, index);
				}
				else
				{
					if ((_iterator != null) && (esp.getActual_name().startsWith("$metadata.") || esp.getActual_name().startsWith("${metadata."))) {
						if (_context.isStandalone()) { // (minor message, while debugging only)
							_context.getHarvestStatus().logMessage("Warning: in actual_name, using global $metadata when iterating", true);
						}
					}
					actualName = getFormattedTextFromField(esp.getActual_name(), field);
				}
				// Since we've specified actual name, we're going to enforce it (unless otherwise specified)
				if (null == actualName) { // failed to get it
					if (null == esp.getCreationCriteriaScript()) {
						if (_context.isStandalone()) { // (minor message, while debugging only)
							_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required actual_name from: ").append(esp.getActual_name()).toString(), true);
						}
						return null;
					}
				}
			}
			// If actualName == null set it equal to disambiguousName
			if (actualName == null) actualName = disambiguatedName;
			e.setActual_name(actualName);
			
			// Entity.type
			String type = null;
			if (esp.getType() != null)
			{
				if (JavaScriptUtils.containsScript(esp.getType()))
				{
					type = (String)getValueFromScript(esp.getType(), field, index);
				}
				else
				{
					type = getFormattedTextFromField(esp.getType(), field);
				}
				// Since we've specified type, we're going to enforce it (unless otherwise specified)
				if (null == type) { // failed to get it
					if (null == esp.getCreationCriteriaScript()) {
						_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required type from: ").append(esp.getType()).toString(), true);
						return null;
					}
				}
			}
			else
			{
				type = "Keyword";
			}
			e.setType(type);
			
			// Entity.index
			String entityIndex = disambiguatedName + "/" + type;
			e.setIndex(entityIndex.toLowerCase());
			
			// Now check if we already exist, discard if so:
			if (_entityMap.contains(e.getIndex())) {
				return null;
			}

			// Entity.dimension
			String dimension = null;
			if (esp.getDimension() != null)
			{
				if (JavaScriptUtils.containsScript(esp.getDimension()))
				{
					dimension = (String)getValueFromScript(esp.getDimension(), field, index);
				}
				else
				{
					dimension = getFormattedTextFromField(esp.getDimension(), field);
				}
				// Since we've specified dimension, we're going to enforce it (unless otherwise specified)
				if (null == dimension) { // failed to get it
					if (null == esp.getCreationCriteriaScript()) {
						_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required dimension from: ").append(esp.getDimension()).toString(), true);
						return null;
					}
				}
			}
			if (null == dimension) {
				try {
					e.setDimension(DimensionUtility.getDimensionByType(type));
				}
				catch (java.lang.IllegalArgumentException ex) {
					e.setDimension(EntityPojo.Dimension.What);									
				}
			}
			else {
				try {
					EntityPojo.Dimension enumDimension = EntityPojo.Dimension.valueOf(dimension);
					if (null == enumDimension) {
						_context.getHarvestStatus().logMessage(new StringBuffer("Invalid dimension: ").append(dimension).toString(), true);
						return null; // (invalid dimension)
					}
					else {
						e.setDimension(enumDimension);
					}
				}
				catch (Exception e2) {
					_context.getHarvestStatus().logMessage(new StringBuffer("Invalid dimension: ").append(dimension).toString(), true);
					return null; // (invalid dimension)					
				}
			}
			
			// Entity.relevance
			String relevance = "0";
			if (esp.getRelevance() != null)
			{
				if (JavaScriptUtils.containsScript(esp.getRelevance()))
				{
					relevance = (String)getValueFromScript(esp.getRelevance(), field, index);
				}
				else
				{
					relevance = getFormattedTextFromField(esp.getRelevance(), field);
				}
				// Since we've specified relevance, we're going to enforce it (unless otherwise specified)
				if (null == relevance) { // failed to get it
					if (null == esp.getCreationCriteriaScript()) {
						_context.getHarvestStatus().logMessage(new StringBuffer("Failed to get required relevance from: ").append(esp.getRelevance()).toString(), true);
						return null;
					}
				}
			}
			try {
				e.setRelevance(Double.parseDouble(relevance));
			}
			catch (Exception e1) {
				this._context.getHarvestStatus().logMessage(e1.getMessage(), true);
				return null;				
			}

			// Entity.sentiment (optional field)
			if (esp.getSentiment() != null)
			{
				String sentiment;
				if (JavaScriptUtils.containsScript(esp.getSentiment()))
				{
					sentiment = (String)getValueFromScript(esp.getSentiment(), field, index);
				}
				else
				{
					sentiment = getFormattedTextFromField(esp.getSentiment(), field);
				}
				// (sentiment is optional, even if specified)
				if (null != sentiment) {
					try {
						double d = Double.parseDouble(sentiment);
						e.setSentiment(d);
						if (null == e.getSentiment()) {
							if (_context.isStandalone()) { // (minor message, while debugging only)
								_context.getHarvestStatus().logMessage(new StringBuffer("Invalid sentiment: ").append(sentiment).toString(), true);
							}							
						}
					}
					catch (Exception e1) {
						this._context.getHarvestStatus().logMessage(e1.getMessage(), true);
						return null;				
					}
				}
			}

			// Entity Link data:
			
			if (esp.getLinkdata() != null)
			{
				
				String linkdata = null;
				if (JavaScriptUtils.containsScript(esp.getLinkdata()))
				{
					linkdata = (String)getValueFromScript(esp.getLinkdata(), field, index);
				}
				else
				{
					linkdata = getFormattedTextFromField(esp.getLinkdata(), field);
				}
				// linkdata i…