PageRenderTime 56ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 1ms

/core/infinit.e.api.server/WEB-INF/src/com/ikanow/infinit/e/api/knowledge/QueryHandler.java

https://github.com/IKANOW/Infinit.e
Java | 3048 lines | 2059 code | 327 blank | 662 comment | 689 complexity | b78d32d20dbf27f2fa21cc4895940870 MD5 | raw file
Possible License(s): BSD-3-Clause

Large files files are truncated, but you can click here to view the full file

  1. /*******************************************************************************
  2. * Copyright 2012, The Infinit.e Open Source Project.
  3. *
  4. * This program is free software: you can redistribute it and/or modify
  5. * it under the terms of the GNU Affero General Public License, version 3,
  6. * as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11. * GNU Affero General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU Affero General Public License
  14. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  15. ******************************************************************************/
  16. package com.ikanow.infinit.e.api.knowledge;
  17. import java.io.IOException;
  18. import java.net.UnknownHostException;
  19. import java.util.ArrayList;
  20. import java.util.Arrays;
  21. import java.util.Calendar;
  22. import java.util.Collection;
  23. import java.util.Date;
  24. import java.util.HashMap;
  25. import java.util.LinkedList;
  26. import java.util.List;
  27. import java.util.Map;
  28. import java.util.Set;
  29. import java.util.TreeSet;
  30. import java.util.concurrent.Semaphore;
  31. import java.util.concurrent.TimeUnit;
  32. import java.util.regex.Pattern;
  33. import org.apache.commons.lang.time.DateUtils;
  34. import org.apache.commons.lang.time.DateFormatUtils;
  35. import org.apache.log4j.Logger;
  36. import org.bson.types.ObjectId;
  37. import org.elasticsearch.action.search.SearchResponse;
  38. import org.elasticsearch.client.action.search.SearchRequestBuilder;
  39. import org.elasticsearch.common.joda.time.Interval;
  40. import org.elasticsearch.common.unit.DistanceUnit;
  41. import org.elasticsearch.index.query.BaseQueryBuilder;
  42. import org.elasticsearch.index.query.BoolFilterBuilder;
  43. import org.elasticsearch.index.query.BoolQueryBuilder;
  44. import org.elasticsearch.index.query.CrossVersionQueryBuilders;
  45. import org.elasticsearch.index.query.CustomFiltersScoreQueryBuilder;
  46. import org.elasticsearch.index.query.FilterBuilders;
  47. import org.elasticsearch.index.query.GeoDistanceFilterBuilder;
  48. import org.elasticsearch.index.query.QueryBuilders;
  49. import org.elasticsearch.index.query.RangeQueryBuilder;
  50. import org.elasticsearch.search.sort.SortOrder;
  51. import com.google.common.collect.ArrayListMultimap;
  52. import com.google.gson.Gson;
  53. import com.google.gson.GsonBuilder;
  54. import com.ikanow.infinit.e.api.knowledge.aliases.AliasLookupTable;
  55. import com.ikanow.infinit.e.api.knowledge.aliases.AliasManager;
  56. import com.ikanow.infinit.e.api.knowledge.processing.AggregationUtils;
  57. import com.ikanow.infinit.e.api.knowledge.processing.QueryDecayFactory;
  58. import com.ikanow.infinit.e.api.knowledge.processing.ScoringUtils;
  59. import com.ikanow.infinit.e.api.social.sharing.ShareHandler;
  60. import com.ikanow.infinit.e.api.utils.PropertiesManager;
  61. import com.ikanow.infinit.e.api.utils.SimpleBooleanParser;
  62. import com.ikanow.infinit.e.api.utils.SocialUtils;
  63. import com.ikanow.infinit.e.data_model.Globals;
  64. import com.ikanow.infinit.e.data_model.api.BasePojoApiMap;
  65. import com.ikanow.infinit.e.data_model.api.ResponsePojo;
  66. import com.ikanow.infinit.e.data_model.api.ResponsePojo.ResponseObject;
  67. import com.ikanow.infinit.e.data_model.api.knowledge.AdvancedQueryPojo;
  68. import com.ikanow.infinit.e.data_model.api.knowledge.StatisticsPojo;
  69. import com.ikanow.infinit.e.data_model.control.DocumentQueueControlPojo;
  70. import com.ikanow.infinit.e.data_model.index.ElasticSearchManager;
  71. import com.ikanow.infinit.e.data_model.index.document.DocumentPojoIndexMap;
  72. import com.ikanow.infinit.e.data_model.interfaces.query.IQueryExtension;
  73. import com.ikanow.infinit.e.data_model.store.DbManager;
  74. import com.ikanow.infinit.e.data_model.store.custom.mapreduce.CustomMapReduceJobPojo;
  75. import com.ikanow.infinit.e.data_model.store.document.AssociationPojo;
  76. import com.ikanow.infinit.e.data_model.store.document.DocumentPojo;
  77. import com.ikanow.infinit.e.data_model.store.document.EntityPojo;
  78. import com.ikanow.infinit.e.data_model.store.feature.entity.EntityFeaturePojo;
  79. import com.ikanow.infinit.e.data_model.store.social.sharing.SharePojo;
  80. import com.ikanow.infinit.e.data_model.utils.GeoOntologyMapping;
  81. import com.mongodb.BasicDBObject;
  82. import com.mongodb.DBCollection;
  83. import com.mongodb.DBCursor;
  84. import com.mongodb.MongoException;
  85. import com.mongodb.ReadPreference;
  86. //
  87. // This code contains all the processing logic for the (beta)
  88. // Advanced Queries
  89. //
  90. //(remove this during active development - want to just depress a deprecation warning but no way of doing this for both 0.19 and 1.0)
  91. //@SuppressWarnings("deprecation")
  92. @SuppressWarnings("all")
  93. public class QueryHandler {
  94. private final StringBuffer _logMsg = new StringBuffer();
  95. private static final Logger _logger = Logger.getLogger(QueryHandler.class);
  96. public QueryHandler() {}
  97. private static Semaphore _concurrentAccessLock = null;
  98. private boolean acquireConcurrentAccessLock() throws InterruptedException {
  99. if (null == _concurrentAccessLock) {
  100. _concurrentAccessLock = new Semaphore(2);
  101. }
  102. return _concurrentAccessLock.tryAcquire(10, TimeUnit.MINUTES);
  103. }
  104. private void releaseConcurrentAccessLock() {
  105. _concurrentAccessLock.release();
  106. }
  107. // Query cache (re-created per request, but there's some static things in here for performance):
  108. private AliasLookupTable _aliasLookup = null;
  109. private LinkedList<AdvancedQueryPojo.QueryTermPojo> _extraFullTextTerms = null;
  110. // (used to allow entity terms to add top level (full text) terms)
  111. private static PropertiesManager _properties = null;
  112. private static com.ikanow.infinit.e.data_model.utils.PropertiesManager _dataModelProps = null;
  113. private static String _aggregationAccuracy = "full";
  114. private static ArrayList<Class<IQueryExtension>> _queryExtensions = null;
  115. private AdvancedQueryPojo.QueryScorePojo _scoringParams;
  116. // (need this here so we can set the adjust param for complex queries)
  117. private static int _replicaSetDistributionRatio = -1;
  118. ////////////////////////////////////////////////////////////////////////
  119. // 0] Top level processing
  120. public ResponsePojo doQuery(String userIdStr, AdvancedQueryPojo query, String communityIdStrList, StringBuffer errorString) throws UnknownHostException, MongoException, IOException, InstantiationException, IllegalAccessException {
  121. if (null == _properties) {
  122. _properties = new PropertiesManager();
  123. _aggregationAccuracy = _properties.getAggregationAccuracy();
  124. _dataModelProps = new com.ikanow.infinit.e.data_model.utils.PropertiesManager();
  125. _replicaSetDistributionRatio = 1 + _dataModelProps.getDocDbReadDistributionRatio();
  126. String[] queryExtensions = _properties.getQueryExtensions();
  127. if (null != queryExtensions) {
  128. _queryExtensions = new ArrayList<Class<IQueryExtension>>(queryExtensions.length);
  129. for (String s: queryExtensions) {
  130. try {
  131. Class<IQueryExtension> queryExtensionClass = (Class<IQueryExtension>) Class.forName(s);
  132. _queryExtensions.add(queryExtensionClass);
  133. }
  134. catch (Exception e) {
  135. _logger.error("Failed to load query extension: " + s, e);
  136. }
  137. catch (Error e) {
  138. _logger.error("Failed to load query extension: " + s, e);
  139. }
  140. }//(end list over query extensions)
  141. if (_queryExtensions.isEmpty()) {
  142. _queryExtensions = null;
  143. }
  144. }//TESTED (see test.QueryExtensionsTestCode)
  145. }
  146. ObjectId queryId = null;
  147. _scoringParams = query.score;
  148. // (NOTE CAN'T ACCESS "query" UNTIL AFTER 0.1 BECAUSE THAT CAN CHANGE IT)
  149. long nSysTime = (_nNow = System.currentTimeMillis());
  150. ResponsePojo rp = new ResponsePojo();
  151. // communityIdList is CSV
  152. String[] communityIdStrs = SocialUtils.getCommunityIds(userIdStr, communityIdStrList);
  153. //(timing)
  154. long nQuerySetupTime = System.currentTimeMillis();
  155. ElasticSearchManager indexMgr = getIndexManager(communityIdStrs);
  156. SearchRequestBuilder searchSettings = indexMgr.getSearchOptions();
  157. StringBuffer querySummary = new StringBuffer();
  158. BaseQueryBuilder queryObj = null;
  159. InternalTempFilterInfo tempFilterInfo = null;
  160. try {
  161. queryObj = getBaseQuery(query, communityIdStrs, communityIdStrList, userIdStr, querySummary);
  162. if (null == queryObj) { // only occurs if has 1 element with ftext starting $cache:
  163. return getSavedQueryInstead(query.qt.get(0).ftext.substring(7), communityIdStrs, query); // (step over cache preamble)
  164. }
  165. tempFilterInfo = getBaseFilter(query, communityIdStrs);
  166. }
  167. catch (Exception e) {
  168. Globals.populateStackTrace(errorString, e);
  169. if (null != e.getCause()) {
  170. errorString.append("[CAUSE=").append(e.getCause().getMessage()).append("]");
  171. Globals.populateStackTrace(errorString, e.getCause());
  172. }
  173. errorString.append(": " + e.getMessage());
  174. return null;
  175. }
  176. //DEBUG
  177. //querySummary.append(new Gson().toJson(query, AdvancedQueryPojo.class));
  178. // 0.4] Pre-Lucene Scoring
  179. // 0.4.1] General
  180. // Different options:
  181. // a] Get the most recent N documents matching the query, score post-query
  182. // b] Get the N highest (Lucene) scoring documents, incorporate significance post-query if desired
  183. // In both cases, N depends on whether significance calculation is taking place (and on the "skip" param)
  184. int nRecordsToOutput = query.output.docs.numReturn;
  185. int nRecordsToSkip = query.output.docs.skip;
  186. int nRecordsToGet = query.score.numAnalyze;
  187. final int nMAXRECORDSTOOUTPUT = 10000;
  188. final int nMAXRECORDSTOGET = 20000;
  189. // Some sanity checking on doc numbers:
  190. if (nRecordsToOutput > nMAXRECORDSTOOUTPUT) { // Upper limit...
  191. errorString.append(": Max # docs to return is 10000.");
  192. return null;
  193. }
  194. if (nRecordsToGet < nRecordsToOutput) {
  195. nRecordsToGet = nRecordsToOutput;
  196. }
  197. else if (nRecordsToGet > nMAXRECORDSTOGET) { // Upper limit...
  198. nRecordsToGet = nMAXRECORDSTOGET; // (we can do something sensible with this so carry on regardless)
  199. }
  200. boolean bUseSignificance = (query.score.sigWeight > 0.0);
  201. boolean bNeedExtraResultsForEnts =
  202. ((query.output.aggregation != null) && (query.output.aggregation.entsNumReturn != null) && (query.output.aggregation.entsNumReturn > 0))
  203. ||
  204. (query.output.docs.enable && (query.output.docs.numReturn > 0) && (query.output.docs.ents) && (query.score.scoreEnts));
  205. if (bUseSignificance || bNeedExtraResultsForEnts) {
  206. // Some logic taken from the original "knowledge/search"
  207. while ( (nRecordsToSkip + nRecordsToOutput > nRecordsToGet) && (nRecordsToGet <= nMAXRECORDSTOGET) )
  208. {
  209. nRecordsToGet += nRecordsToGet;
  210. }
  211. if (nRecordsToGet > nMAXRECORDSTOGET) {
  212. errorString.append(": Can only skip through to 20000 documents.");
  213. return null;
  214. }
  215. searchSettings.setSize(nRecordsToGet);
  216. //TESTED
  217. }
  218. else if (query.output.docs.enable) { // In this case we just need the minimum number of records
  219. // (whether searching by date or by relevance)
  220. searchSettings.setFrom(nRecordsToSkip);
  221. nRecordsToSkip = 0; // (so it isn't double counted in the processing module)
  222. nRecordsToGet = nRecordsToOutput;
  223. searchSettings.setSize(nRecordsToGet);
  224. //TESTED
  225. }
  226. else { // In thise case we're just outputting aggregations, and not even ones that come from the docs
  227. nRecordsToGet = 0; // (use this variable everywhere where we care about bring docs back either to output or for suitable aggregation)
  228. searchSettings.setSize(0);
  229. }
  230. // Sort on score if relevance is being used
  231. if (nRecordsToGet > 0) {
  232. if (query.score.relWeight > 0.0) { // (b) above
  233. // Using score is default, nothing to do
  234. }
  235. else { // (a) above
  236. // Debug code, if rel weight negative then use date to check Lucene score is better...
  237. if (query.score.relWeight < 0.0) {
  238. query.score.relWeight = -query.score.relWeight;
  239. }
  240. // Set Lucene to order:
  241. searchSettings.addSort(DocumentPojo.publishedDate_, SortOrder.DESC);
  242. }//TOTEST
  243. }//(if docs aren't enabled, don't need to worry about sorting)
  244. // 0.4.2] Prox scoring (needs to happen after [0.3]
  245. // Add proximity scoring:
  246. boolean bLowAccuracyDecay = false;
  247. if ((nRecordsToGet > 0) || (null == _scoringParams.adjustAggregateSig) || _scoringParams.adjustAggregateSig) {
  248. // (ie if we're getting docs or applying scores to entities)
  249. if (!_aggregationAccuracy.equals("full")) {
  250. bLowAccuracyDecay = true;
  251. }
  252. queryObj = addProximityBasedScoring(queryObj, searchSettings, query.score, tempFilterInfo.parentFilterObj, bLowAccuracyDecay);
  253. if (null == _scoringParams.adjustAggregateSig) { // auto-decide .. if ftext is set and is non-trivial
  254. if ((null != query.score.timeProx) || (null != query.score.geoProx)) {
  255. // (These are set to null above if badly formed)
  256. _scoringParams.adjustAggregateSig = true;
  257. }
  258. }
  259. }// (else not worth the effort)
  260. // 0.4.3] Source weightings (if any)
  261. queryObj = applyManualWeights(queryObj, query.score);
  262. // 0.5] Pre-lucene output options
  263. // only return the id field and score
  264. // (Both _id and score come back as default options, SearchHit:: getId and getScore, don't need anything else)
  265. // Facets
  266. // (These are needed for the case where we need to perform aggregations manually)
  267. Integer manualEntsNumReturn = null;
  268. Integer manualEventsNumReturn = null;
  269. Integer manualFactsNumReturn = null;
  270. Integer manualGeoNumReturn = null;
  271. //DEBUG
  272. //System.out.println(new Gson().toJson(query.output.aggregation));
  273. if ((null != query.output.aggregation) && (null != query.output.aggregation.raw)) { // Like query, specify raw aggregation (Facets)
  274. // Gross raw handling for facets
  275. if ((null != query.raw) && (null != query.raw.query)) {
  276. // Don't currently support raw query and raw facets because I can't work out how to apply
  277. // the override on group/source!
  278. errorString.append(": Not currently allowed raw query and raw facets");
  279. return null;
  280. }
  281. else { // Normal code
  282. searchSettings.setFacets(query.output.aggregation.raw.getBytes());
  283. }
  284. }
  285. else { // Apply various aggregation (=="facet") outputs to searchSettings
  286. boolean bSpecialCase = (null != query.raw) && (null != query.raw.query);
  287. if (!_aggregationAccuracy.equals("full")) {
  288. if (null != query.output.aggregation) {
  289. if (_aggregationAccuracy.equals("low")) {
  290. manualEntsNumReturn = query.output.aggregation.entsNumReturn;
  291. manualEventsNumReturn = query.output.aggregation.eventsNumReturn;
  292. manualFactsNumReturn = query.output.aggregation.factsNumReturn;
  293. manualGeoNumReturn = query.output.aggregation.geoNumReturn;
  294. }
  295. query.output.aggregation.entsNumReturn = null;
  296. query.output.aggregation.eventsNumReturn = null;
  297. query.output.aggregation.factsNumReturn = null;
  298. query.output.aggregation.geoNumReturn = null;
  299. // (allow time aggregation)
  300. // (allow source aggregation)
  301. }
  302. }
  303. AggregationUtils.parseOutputAggregation(query.output.aggregation, _aliasLookup,
  304. tempFilterInfo.entityTypeFilterStrings, tempFilterInfo.assocVerbFilterStrings,
  305. searchSettings, bSpecialCase?tempFilterInfo.parentFilterObj:null);
  306. // In partial accuracy case, restore aggregation
  307. if (null != manualEntsNumReturn) {
  308. query.output.aggregation.entsNumReturn = manualEntsNumReturn;
  309. }
  310. if (null != manualEventsNumReturn) {
  311. query.output.aggregation.eventsNumReturn = manualEventsNumReturn;
  312. }
  313. if (null != manualFactsNumReturn) {
  314. query.output.aggregation.factsNumReturn = manualFactsNumReturn;
  315. }
  316. if (null != manualGeoNumReturn) {
  317. query.output.aggregation.geoNumReturn = manualGeoNumReturn;
  318. }
  319. //TESTED
  320. }
  321. //TESTED x2
  322. //(timing)
  323. nQuerySetupTime = System.currentTimeMillis() - nQuerySetupTime;
  324. // 0.6] Perform Lucene query
  325. // 0.6.1: query extensions: pre-query hook
  326. ArrayList<IQueryExtension> queryExtensions = null;
  327. if (null != _queryExtensions) {
  328. queryId = new ObjectId();
  329. queryExtensions = new ArrayList<IQueryExtension>(_queryExtensions.size());
  330. for (Class<IQueryExtension> queryExtensionClass: _queryExtensions) {
  331. // Don't catch any exceptions thrown here - let it bubble upwards
  332. IQueryExtension queryExtension = queryExtensionClass.newInstance();
  333. queryExtension.preQueryActivities(queryId, query, communityIdStrs);
  334. queryExtensions.add(queryExtension);
  335. }
  336. }//TESTED (see test.QueryExtensionsTestCode)
  337. // 0.6.2: the main query
  338. if ((null != query.explain) && query.explain) { // (for diagnostic - will return lucene explanation)
  339. searchSettings.setExplain(true);
  340. }
  341. SearchResponse queryResults = null;
  342. if ((null != query.raw) && (null != query.raw.query))
  343. {
  344. // (Can bypass all other settings)
  345. searchSettings.setQuery(query.raw.query);
  346. queryResults = indexMgr.doQuery(null, tempFilterInfo.parentFilterObj, searchSettings);
  347. }//TESTED '{ "raw": { "match_all": {} } }'
  348. else
  349. {
  350. // Where I can, use the source filter as part of the query so that
  351. // facets will apply to query+filter, not just filter
  352. queryObj = QueryBuilders.boolQuery().must(queryObj).must(QueryBuilders.constantScoreQuery(tempFilterInfo.parentFilterObj).boost(0.0F));
  353. queryResults = indexMgr.doQuery(queryObj, null, searchSettings);
  354. }//TESTED '{}' etc
  355. long nLuceneTime = queryResults.getTookInMillis();
  356. // 0.7] Lucene scores
  357. long nProcTime = 0;
  358. long nProcTime_tmp = System.currentTimeMillis();
  359. StatisticsPojo stats = new StatisticsPojo();
  360. stats.found = queryResults.getHits().getTotalHits();
  361. stats.start = (long)nRecordsToSkip;
  362. if (nRecordsToGet > 0) {
  363. stats.setScore(queryResults.getHits(), (null != query.score.geoProx)||(null != query.score.timeProx), (null != query.explain) && query.explain);
  364. }
  365. //DEBUG
  366. //System.out.println(new Gson().toJson(queryResults));
  367. nProcTime += (System.currentTimeMillis() - nProcTime_tmp);
  368. // 0.8] Get data from Mongo + handle scoring
  369. //(timing)
  370. long nMongoTime = System.currentTimeMillis();
  371. List<BasicDBObject> docs = null;
  372. //(aggregation)
  373. LinkedList<BasicDBObject> lowAccuracyAggregatedEntities = null; // (always low accuracy)
  374. LinkedList<BasicDBObject> standaloneEvents = null;
  375. LinkedList<BasicDBObject> lowAccuracyAggregatedEvents = null;
  376. LinkedList<BasicDBObject> lowAccuracyAggregatedFacts = null;
  377. AggregationUtils.GeoContainer lowAccuracyAggregatedGeo = null;
  378. AggregationUtils.GeoContainer extraAliasAggregatedGeo = null;
  379. ScoringUtils scoreStats = null;
  380. if (null != stats.getIds()) {
  381. DBCursor docs0 = this.getDocIds(DbManager.getDocument().getMetadata(), stats.getIds(), nRecordsToGet, query.output, query.score);
  382. nMongoTime = System.currentTimeMillis() - nMongoTime;
  383. nProcTime_tmp = System.currentTimeMillis();
  384. // Entity aggregation (CURRENTLY ALWAYS LOW AGGREGATION):
  385. if ((null != query.output.aggregation) && (null != query.output.aggregation.entsNumReturn) && (query.output.aggregation.entsNumReturn > 0)) {
  386. lowAccuracyAggregatedEntities = new LinkedList<BasicDBObject>();
  387. }
  388. // Standalone events:
  389. if ((query.output.docs != null) && (query.output.docs.eventsTimeline != null) && query.output.docs.eventsTimeline) {
  390. standaloneEvents = new LinkedList<BasicDBObject>();
  391. }
  392. // Low accuracy aggregations:
  393. if ((null != manualEventsNumReturn) && (manualEventsNumReturn > 0)) {
  394. lowAccuracyAggregatedEvents = new LinkedList<BasicDBObject>();
  395. }
  396. if ((null != manualFactsNumReturn) && (manualFactsNumReturn > 0)) {
  397. lowAccuracyAggregatedFacts = new LinkedList<BasicDBObject>();
  398. }
  399. if ((null != manualGeoNumReturn) && (manualGeoNumReturn > 0)) {
  400. lowAccuracyAggregatedGeo = new AggregationUtils.GeoContainer();
  401. }
  402. else if ((null != query.output.aggregation) && (null != query.output.aggregation.geoNumReturn) && (query.output.aggregation.geoNumReturn > 0))
  403. {
  404. // (only if not using low accuracy aggregation ... otherwise it all gets dumped in lowAccuracyAggregatedGeo)
  405. extraAliasAggregatedGeo = new AggregationUtils.GeoContainer();
  406. }
  407. scoreStats = new ScoringUtils();
  408. try {
  409. boolean lockAcquired = true;
  410. try {
  411. lockAcquired = this.acquireConcurrentAccessLock();
  412. } catch (InterruptedException e) {
  413. //(that's fine just carry on)
  414. lockAcquired = false;
  415. }
  416. if (!lockAcquired) {
  417. rp.setResponse(new ResponseObject("Query", false, "Query engine busy, please try again later."));
  418. return rp;
  419. }
  420. scoreStats.setAliasLookupTable(_aliasLookup);
  421. docs = scoreStats.calcTFIDFAndFilter(DbManager.getDocument().getMetadata(),
  422. docs0, query.score, query.output, stats, bLowAccuracyDecay,
  423. nRecordsToSkip, nRecordsToOutput,
  424. communityIdStrs,
  425. tempFilterInfo.entityTypeFilterStrings, tempFilterInfo.assocVerbFilterStrings,
  426. standaloneEvents,
  427. lowAccuracyAggregatedEntities,
  428. lowAccuracyAggregatedGeo, extraAliasAggregatedGeo,
  429. lowAccuracyAggregatedEvents, lowAccuracyAggregatedFacts);
  430. }
  431. finally {
  432. scoreStats.clearAsMuchMemoryAsPossible();
  433. this.releaseConcurrentAccessLock();
  434. }
  435. nProcTime += (System.currentTimeMillis() - nProcTime_tmp);
  436. }
  437. else {
  438. nMongoTime = 0;
  439. }
  440. //TESTED (all queries)
  441. // 0.9] Output:
  442. rp.setResponse(new ResponseObject("Query", true, querySummary.toString()));
  443. // 0.9.1] Stats:
  444. stats.resetArrays();
  445. rp.setStats(stats); // (only actually uses the response pojo, but get rid of big fields anyway...)
  446. // 0.9.2] Facets:
  447. if (null != lowAccuracyAggregatedEntities) { // Entity aggregation
  448. rp.setEntities(lowAccuracyAggregatedEntities);
  449. }
  450. if (null != standaloneEvents) {
  451. rp.setEventsTimeline(standaloneEvents);
  452. }
  453. if (null != lowAccuracyAggregatedGeo) {
  454. rp.setGeo(lowAccuracyAggregatedGeo.geotags, (int)lowAccuracyAggregatedGeo.maxCount, (int)lowAccuracyAggregatedGeo.minCount);
  455. }
  456. if (null != lowAccuracyAggregatedEvents) {
  457. rp.setEvents(lowAccuracyAggregatedEvents);
  458. }
  459. if (null != lowAccuracyAggregatedFacts) {
  460. rp.setFacts(lowAccuracyAggregatedFacts);
  461. }
  462. if ((null != query.output.aggregation) && (null != query.output.aggregation.raw)) {
  463. rp.setFacets(queryResults.getFacets().facetsAsMap());
  464. }
  465. else if ((null != queryResults.getFacets()) && (null != queryResults.getFacets().getFacets())) { // "Logical" aggregation
  466. if (0.0 == query.score.sigWeight) {
  467. scoreStats = null; // (don't calculate event/fact aggregated significance if it's not wanted)
  468. }
  469. AggregationUtils.loadAggregationResults(rp, queryResults.getFacets().getFacets(), query.output.aggregation, scoreStats, _aliasLookup, tempFilterInfo.entityTypeFilterStrings, tempFilterInfo.assocVerbFilterStrings, extraAliasAggregatedGeo);
  470. } // (end facets not overwritten)
  471. scoreStats = null; // (now definitely never need scoreStats)
  472. // 0.9.3] Documents
  473. if (query.output.docs.enable) {
  474. if ((null != docs) && (docs.size() > 0)) {
  475. rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
  476. }
  477. else { // (ensure there's always an empty list)
  478. docs = new ArrayList<BasicDBObject>(0);
  479. rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
  480. }
  481. }
  482. else { // (ensure there's always an empty list)
  483. docs = new ArrayList<BasicDBObject>(0);
  484. rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
  485. }
  486. // 0.9.4] query extensions: post-query hook
  487. if (null != queryExtensions) {
  488. for (IQueryExtension queryExtension: queryExtensions) {
  489. // Don't catch any exceptions thrown here - let it bubble upwards
  490. queryExtension.postQueryActivities(queryId, docs, rp);
  491. }
  492. }//TESTED (see test.QueryExtensionsTestCode)
  493. // 0.9.5] Timing/logging
  494. long nTotalTime = System.currentTimeMillis() - nSysTime;
  495. rp.getResponse().setTime(nTotalTime);
  496. _logMsg.setLength(0);
  497. _logMsg.append("knowledge/query querylen=").append(querySummary.length());
  498. _logMsg.append(" query=").append(querySummary.toString());
  499. _logMsg.append(" userid=").append(userIdStr);
  500. _logMsg.append(" groups=").append(communityIdStrList);
  501. _logMsg.append(" found=").append(stats.found);
  502. _logMsg.append(" luceneTime=").append(nLuceneTime).append(" ms");
  503. _logMsg.append(" setupTime=").append(nQuerySetupTime).append(" ms");
  504. _logMsg.append(" procTime=").append(nProcTime).append(" ms");
  505. _logMsg.append(" mongoTime=").append(nMongoTime).append(" ms");
  506. _logMsg.append(" time=").append(nTotalTime).append(" ms");
  507. _logger.info(_logMsg.toString());
  508. //DEBUG
  509. //System.out.println(_logMsg.toString());
  510. // Exceptions percolate up to the resource and are handled there...
  511. return rp;
  512. }
  513. ////////////////////////////////////////////////////////////////////////
  514. // Utility version of the above query call - just converts the advanced query pojo into an elasticsearch object that can
  515. // be queried
  516. public static class QueryInfo {
  517. public ElasticSearchManager indexMgr;
  518. public BaseQueryBuilder queryObj;
  519. public String querySummary;
  520. }
  521. public QueryInfo convertInfiniteQuery(AdvancedQueryPojo query, String[] communityIdStrs, String userIdStr) {
  522. // Fill in the blanks (a decent attempt has been made to fill out the blanks inside these options)
  523. if (null == query.input) {
  524. query.input = new AdvancedQueryPojo.QueryInputPojo();
  525. }
  526. if (null == query.score) {
  527. query.score = new AdvancedQueryPojo.QueryScorePojo();
  528. }
  529. if (null == query.output) {
  530. query.output = new AdvancedQueryPojo.QueryOutputPojo();
  531. }
  532. if (null == query.output.docs) { // (Docs are sufficiently important we'll make sure they're always present)
  533. query.output.docs = new AdvancedQueryPojo.QueryOutputPojo.DocumentOutputPojo();
  534. }
  535. // Other intialization
  536. _nNow = System.currentTimeMillis();
  537. // Now onto the logic:
  538. QueryInfo queryInfo = new QueryInfo();
  539. StringBuffer sb = new StringBuffer(userIdStr);
  540. for (String sCommunityId: communityIdStrs) {
  541. sb.append(',').append(sCommunityId);
  542. }
  543. queryInfo.indexMgr = getIndexManager(communityIdStrs);
  544. StringBuffer info = new StringBuffer();
  545. queryInfo.queryObj = getBaseQuery(query, communityIdStrs, sb.toString(), userIdStr, info);
  546. queryInfo.querySummary = info.toString();
  547. InternalTempFilterInfo tempFilterInfo = getBaseFilter(query, communityIdStrs);
  548. queryInfo.queryObj = QueryBuilders.boolQuery().must(queryInfo.queryObj).
  549. must(QueryBuilders.constantScoreQuery(tempFilterInfo.parentFilterObj).boost(0.0F));
  550. return queryInfo;
  551. }//TOTEST
  552. ////////////////////////////////////////////////////////////////////////
  553. //0b] QUERY BREAKDOWN
  554. ////////////////////////////////////////////////////////////////////////
  555. // 0.b.1) indexes
  556. private ElasticSearchManager getIndexManager(String[] communityIdStrs)
  557. {
  558. // Create a multi-index to check against all relevant shards:
  559. StringBuffer sb = new StringBuffer(DocumentPojoIndexMap.globalDocumentIndexCollection_);
  560. sb.append(',').append(DocumentPojoIndexMap.manyGeoDocumentIndexCollection_);
  561. for (String sCommunityId: communityIdStrs) {
  562. sb.append(',').append("docs_").append(sCommunityId);
  563. }
  564. sb.append('/').append(DocumentPojoIndexMap.documentType_);
  565. ElasticSearchManager indexMgr = ElasticSearchManager.getIndex(sb.toString());
  566. return indexMgr;
  567. }//TESTED (cut and paste from original code)
  568. ////////////////////////////////////////////////////////////////////////
  569. // 0.b.1) filter
  570. private static class InternalTempFilterInfo {
  571. BoolFilterBuilder parentFilterObj;
  572. String[] entityTypeFilterStrings;
  573. String[] assocVerbFilterStrings;
  574. }
  575. private InternalTempFilterInfo getBaseFilter(AdvancedQueryPojo query, String communityIdStrs[])
  576. {
  577. BoolFilterBuilder parentFilterObj =
  578. FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(DocumentPojo.communityId_, communityIdStrs));
  579. BoolFilterBuilder sourceFilter = this.parseSourceManagement(query.input);
  580. if (null != sourceFilter) {
  581. parentFilterObj = parentFilterObj.must(sourceFilter);
  582. }//TESTED
  583. // 0.2] Output filtering
  584. // Output filters: parse (also used by aggregation, scoring)
  585. String[] entityTypeFilterStrings = null;
  586. String[] assocVerbFilterStrings = null;
  587. if ((null != query.output) && (null != query.output.filter)) {
  588. if (null != query.output.filter.entityTypes) {
  589. entityTypeFilterStrings = query.output.filter.entityTypes;
  590. if (0 == entityTypeFilterStrings.length) {
  591. entityTypeFilterStrings = null;
  592. }
  593. else if ((1 == entityTypeFilterStrings.length) && (entityTypeFilterStrings[0].isEmpty())) {
  594. entityTypeFilterStrings = null;
  595. }
  596. // (note this is important because it means we can always check entityTypeFilterStrings[0].getCharAt(0) for -ve filtering)
  597. }
  598. if (null != query.output.filter.assocVerbs) {
  599. assocVerbFilterStrings = query.output.filter.assocVerbs;
  600. if (0 == assocVerbFilterStrings.length) {
  601. assocVerbFilterStrings = null;
  602. }
  603. else if ((1 == assocVerbFilterStrings.length) && (assocVerbFilterStrings[0].isEmpty())) {
  604. assocVerbFilterStrings = null;
  605. }
  606. // (note this is important because it means we can always check assocVerbFilterStrings[0].getCharAt(0) for -ve filtering)
  607. }
  608. }
  609. // Now apply output filters to query
  610. BoolFilterBuilder outputFilter = this.parseOutputFiltering(entityTypeFilterStrings, assocVerbFilterStrings);
  611. if (null != outputFilter) {
  612. parentFilterObj = parentFilterObj.must(outputFilter);
  613. }
  614. //TESTED
  615. InternalTempFilterInfo out = new InternalTempFilterInfo();
  616. out.parentFilterObj = parentFilterObj;
  617. out.entityTypeFilterStrings = entityTypeFilterStrings;
  618. out.assocVerbFilterStrings = assocVerbFilterStrings;
  619. return out;
  620. }//TESTED (cut/paste from original code)
  621. ////////////////////////////////////////////////////////////////////////
  622. // 0.b.2] Query
  623. // (if it returns null then call getSavedQueryInstead instead)
  624. private BaseQueryBuilder getBaseQuery(AdvancedQueryPojo query, String communityIdStrs[], String communityIdStrList, String userIdStr, StringBuffer querySummary)
  625. {
  626. // Intialize alias if so required:
  627. if ((null == query.expandAlias) || query.expandAlias) {
  628. AliasManager aliasManager = AliasManager.getAliasManager();
  629. if (null != aliasManager) {
  630. _aliasLookup = aliasManager.getAliasLookupTable(null, communityIdStrs, null, userIdStr);
  631. }
  632. }
  633. // (end initialize index)
  634. BaseQueryBuilder queryObj = null;
  635. // 0.1] Input data (/filtering)
  636. if (null != query.input.name) { // This is actually a share id visible to this user
  637. query = getStoredQueryArtefact(query.input.name, query, userIdStr);
  638. }
  639. // 0.3] Query terms
  640. int nQueryElements = 0;
  641. if (null != query.qt) {
  642. nQueryElements = query.qt.size();
  643. if ((1 == nQueryElements) && (null != query.qt.get(0).ftext) && (query.qt.get(0).ftext.startsWith("$cache:"))) {
  644. return null;
  645. }
  646. if (nQueryElements > 0) { // NORMAL CASE
  647. this.handleEntityExpansion(DbManager.getFeature().getEntity(), query.qt, userIdStr, communityIdStrList);
  648. BaseQueryBuilder queryElements[] = new BaseQueryBuilder[nQueryElements];
  649. StringBuffer sQueryElements[] = new StringBuffer[nQueryElements];
  650. for (int i = 0; i < nQueryElements; ++i) {
  651. _extraFullTextTerms = null;
  652. queryElements[i] = this.parseQueryTerm(query.qt.get(i), (sQueryElements[i] = new StringBuffer()));
  653. // Extra full text terms generated by aliasing:
  654. if (null != _extraFullTextTerms) {
  655. BoolQueryBuilder extraTerms = QueryBuilders.boolQuery().should(queryElements[i]);
  656. StringBuffer discard = new StringBuffer(); // (we already have added the info the query elements)
  657. for (AdvancedQueryPojo.QueryTermPojo qtExtra: _extraFullTextTerms) {
  658. extraTerms = extraTerms.should(this.parseQueryTerm(qtExtra, discard));
  659. }
  660. queryElements[i] = extraTerms;
  661. _extraFullTextTerms = null; // (reset ready for next term...)
  662. }//TESTED
  663. }//end loop over query terms
  664. queryObj = this.parseLogic(query.logic, queryElements, sQueryElements, querySummary);
  665. if (null == queryObj) { //error parsing logic
  666. throw new RuntimeException("Error parsing logic");
  667. }
  668. }
  669. else { //(QT exists but doesn't have any elements)
  670. queryObj = QueryBuilders.matchAllQuery();
  671. querySummary.append('*');
  672. }
  673. }//TESTED
  674. else {
  675. queryObj = QueryBuilders.matchAllQuery();
  676. querySummary.append('*');
  677. } //(QT not specified)
  678. return queryObj;
  679. }//TESTED (cut/paste from original code)
  680. ////////////////////////////////////////////////////////////////////////
  681. //1] QUERY UTILITIES
  682. ////////////////////////////////////////////////////////////////////////
  683. // 1.0] Stored queries/datasets
  684. // Saved queries (ie the entire dataset)
  685. private ResponsePojo getSavedQueryInstead(String storedQueryNameOrId, String[] communityIdStrs, AdvancedQueryPojo query) {
  686. ResponsePojo rp = null;
  687. ObjectId oid = null;
  688. BasicDBObject jobQuery = null;
  689. try {
  690. oid = new ObjectId(storedQueryNameOrId);
  691. jobQuery = new BasicDBObject(CustomMapReduceJobPojo._id_, oid);
  692. }
  693. catch (Exception e) {
  694. jobQuery = new BasicDBObject(CustomMapReduceJobPojo.jobtitle_, storedQueryNameOrId);
  695. }
  696. CustomMapReduceJobPojo savedJob = CustomMapReduceJobPojo.fromDb(DbManager.getCustom().getLookup().findOne(jobQuery), CustomMapReduceJobPojo.class);
  697. if (null != savedJob) { // Is this even a saved job?
  698. if (null != savedJob.jarURL) {
  699. savedJob = null;
  700. }
  701. }
  702. if (null != savedJob) { // Authorization
  703. boolean auth = false;
  704. String communityIdStrList = Arrays.toString(communityIdStrs);
  705. for (ObjectId commId: savedJob.communityIds) {
  706. if (communityIdStrList.contains(commId.toString())) {
  707. auth = true;
  708. break;
  709. }
  710. }
  711. if (!auth) {
  712. savedJob = null;
  713. }
  714. if (null == savedJob) {
  715. throw new RuntimeException("Can't find saved query, or is a custom job not a query, or authorization error");
  716. }
  717. // OK go get the results of the job
  718. DBCollection coll = DbManager.getCollection(savedJob.getOutputDatabase(), savedJob.outputCollection);
  719. BasicDBObject result = (BasicDBObject) coll.findOne(); // (at some point support multiple saved queries)
  720. if (null == result) {
  721. throw new RuntimeException("Saved query is empty");
  722. }
  723. BasicDBObject apiResultToConvert = (BasicDBObject) result.get("value");
  724. if (null == apiResultToConvert) {
  725. throw new RuntimeException("Saved query has invalid format");
  726. }
  727. rp = ResponsePojo.fromDb(apiResultToConvert);
  728. }
  729. else if (null != oid) { // Support new user/doc queues
  730. SharePojo share = SharePojo.fromDb(DbManager.getSocial().getShare().findOne(jobQuery), SharePojo.class);
  731. if ((null == share) || (null == share.getShare()) ||
  732. (!share.getType().equals(DocumentQueueControlPojo.UserQueue) && !share.getType().equals(DocumentQueueControlPojo.SavedQueryQueue))
  733. )
  734. {
  735. throw new RuntimeException("Can't find saved query, or is a custom job not a query, or authorization error");
  736. }
  737. else { // share.share is a DocumentQueueControlPojo
  738. DocumentQueueControlPojo queue = DocumentQueueControlPojo.fromApi(share.getShare(), DocumentQueueControlPojo.class);
  739. BasicDBObject docQuery1 = new BasicDBObject(DocumentPojo._id_, new BasicDBObject(DbManager.in_, queue.getQueueList()));
  740. BasicDBObject docQuery2 = new BasicDBObject(DocumentPojo.updateId_, new BasicDBObject(DbManager.in_, queue.getQueueList()));
  741. BasicDBObject docQuery = new BasicDBObject(DbManager.or_, Arrays.asList(docQuery1, docQuery2));
  742. DBCursor dbc = DbManager.getDocument().getMetadata().find(docQuery).limit(query.score.numAnalyze);
  743. ScoringUtils scoreStats = new ScoringUtils();
  744. List<BasicDBObject> docs = null;
  745. StatisticsPojo stats = new StatisticsPojo();
  746. stats.setSavedScores(query.output.docs.skip, dbc.count());
  747. try {
  748. boolean lockAcquired = true;
  749. try {
  750. lockAcquired = this.acquireConcurrentAccessLock();
  751. } catch (InterruptedException e) {
  752. //(that's fine just carry on)
  753. lockAcquired = false;
  754. }
  755. if (!lockAcquired) {
  756. rp.setResponse(new ResponseObject("Query", false, "Query engine busy, please try again later."));
  757. return rp;
  758. }
  759. scoreStats.setAliasLookupTable(_aliasLookup);
  760. docs = scoreStats.calcTFIDFAndFilter(DbManager.getDocument().getMetadata(),
  761. dbc, query.score, query.output, stats, false,
  762. query.output.docs.skip, query.output.docs.numReturn,
  763. communityIdStrs,
  764. null, null,
  765. null,
  766. null,
  767. null, null,
  768. null, null);
  769. }
  770. finally {
  771. scoreStats.clearAsMuchMemoryAsPossible();
  772. this.releaseConcurrentAccessLock();
  773. }
  774. rp = new ResponsePojo();
  775. rp.setResponse(new ResponseObject("Query", true, "Saved Query: " + share.getTitle()));
  776. rp.setStats(stats);
  777. if ((null != docs) && (docs.size() > 0)) {
  778. rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
  779. }
  780. else { // (ensure there's always an empty list)
  781. docs = new ArrayList<BasicDBObject>(0);
  782. rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
  783. }
  784. }//end if user or saved query queue
  785. }
  786. return rp;
  787. }//TESTED
  788. // Stored queries (ie just the query JSON)
  789. static AdvancedQueryPojo getStoredQueryArtefact(String shareIdStr, AdvancedQueryPojo query, String userIdStr) {
  790. ResponsePojo rp2 = new ShareHandler().getShare(userIdStr, shareIdStr, true);
  791. if ((null != rp2.getData() || !rp2.getResponse().isSuccess())) {
  792. SharePojo share = (SharePojo) rp2.getData();
  793. if (null != share) {
  794. if (share.getType().equalsIgnoreCase("dataset")) {
  795. query.input = new com.google.gson.Gson().fromJson(share.getShare(), AdvancedQueryPojo.QueryInputPojo.class);
  796. }
  797. else if (share.getType().equalsIgnoreCase("query")) {
  798. query = new com.google.gson.Gson().fromJson(share.getShare(), AdvancedQueryPojo.class);
  799. }
  800. else { // Unrecognized share
  801. throw new RuntimeException("Unexpected share type: " + share.getType());
  802. }
  803. }
  804. else {
  805. throw new RuntimeException("Invalid return from share: " + rp2.getData().toString());
  806. }
  807. }
  808. else {
  809. throw new RuntimeException(rp2.getResponse().getMessage());
  810. }
  811. return query;
  812. }
  813. ////////////////////////////////////////////////////////////////////////
  814. // 1.1] Source management utility
  815. BoolFilterBuilder parseSourceManagement(AdvancedQueryPojo.QueryInputPojo input) {
  816. BoolFilterBuilder sourceFilter = null;
  817. if ((null != input.tags) || (null != input.typeAndTags)
  818. || (null != input.sources))
  819. {
  820. sourceFilter = FilterBuilders.boolFilter();
  821. }//TESTED
  822. if (null != input.tags) {
  823. sourceFilter = sourceFilter.should(FilterBuilders.termsFilter(DocumentPojo.tags_, input.tags.toArray()));
  824. }//TESTED '{ "input": { "tags": [ "healthcare", "cyber" ] } }'
  825. if (null != input.typeAndTags) {
  826. BoolFilterBuilder typeAndTagFilter = FilterBuilders.boolFilter();
  827. for (AdvancedQueryPojo.QueryInputPojo.TypeAndTagTermPojo tt: input.typeAndTags) {
  828. if (null != tt.tags) {
  829. typeAndTagFilter = typeAndTagFilter.should(
  830. FilterBuilders.boolFilter().must(FilterBuilders.termFilter(DocumentPojo.mediaType_, tt.type)).
  831. must(FilterBuilders.termsFilter(DocumentPojo.tags_, tt.tags.toArray())));
  832. }
  833. else {
  834. typeAndTagFilter = typeAndTagFilter.should(FilterBuilders.termFilter(DocumentPojo.mediaType_, tt.type));
  835. }
  836. }
  837. sourceFilter = sourceFilter.should(typeAndTagFilter);
  838. }//TESTED '{ "input": { "typeAndTags": [ { "type": "Social" }, { "type": "Video", "tags": [ "education", "MIT" ] } ] } }'
  839. if (null != input.sources) {
  840. if ((null == input.srcInclude) || input.srcInclude) {
  841. sourceFilter = sourceFilter.should(FilterBuilders.termsFilter(DocumentPojo.sourceKey_, input.sources.toArray()));
  842. }
  843. else {
  844. sourceFilter = sourceFilter.mustNot(FilterBuilders.termsFilter(DocumentPojo.sourceKey_, input.sources.toArray()));
  845. }
  846. }//TESTED '{ "input": { "srcInclude": false, "sources": [ "http.twitter.com.statuses.public_timeline.atom", "http.gdata.youtube.com.feeds.base.users.mit.uploads.alt=rss.v=2.orderby=published.client=ytapi-youtube-profile" ] } }'
  847. //(also "srcInclude" not set - checked got the complement of the result)
  848. return sourceFilter;
  849. }
  850. ////////////////////////////////////////////////////////////////////////
  851. // 1.X2] Output filter parsing
  852. private BoolFilterBuilder addNegativeSelectorToFilter(EntityFeaturePojo docDiscardAlias, BoolFilterBuilder outputFilter, int recursionLevel) {
  853. if ((null != docDiscardAlias.getAlias()) && !docDiscardAlias.getAlias().isEmpty()) {
  854. if (null == outputFilter) {
  855. outputFilter = FilterBuilders.boolFilter();
  856. }
  857. outputFilter = outputFilter.mustNot(FilterBuilders.nestedFilter(DocumentPojo.entities_,
  858. FilterBuilders.termsFilter(EntityPojo.docQuery_index_, docDiscardAlias.getAlias().toArray())));
  859. if (recursionLevel <= 1) { // (only go two deep for now)
  860. for (String aliasIndex: docDiscardAlias.getAlias()) {
  861. EntityFeaturePojo docDiscardSubAlias = _aliasLookup.getAliases(aliasIndex);
  862. if (null != docDiscardSubAlias) {
  863. outputFilter = addNegativeSelectorToFilter(docDiscardSubAlias, outputFilter, 1 + recursionLevel);
  864. }
  865. }//TESTED
  866. }
  867. }//TESTED (by hand)
  868. if (null != docDiscardAlias.getSemanticLinks()) { // (recall: we've abused this field for text queries)
  869. for (String textQuery: docDiscardAlias.getSemanticLinks()) {
  870. //(probably not a very efficient construct, but nothing about this is efficient, just functional, so we'll leave it for now)
  871. outputFilter = outputFilter.mustNot(FilterBuilders.queryFilter(
  872. CrossVersionQueryBuilders.matchPhraseQuery(DocumentPojo.fullText_, textQuery)));
  873. outputFilter = outputFilter.mustNot(FilterBuilders.queryFilter(
  874. CrossVersionQueryBuilders.matchPhraseQuery("_all", textQuery)));
  875. }
  876. }//TESTED (by hand)
  877. return outputFilter;
  878. }//TESTED
  879. BoolFilterBuilder parseOutputFiltering(String[] entityTypeFilterStrings, String[] assocVerbFilterStrings)
  880. {
  881. BoolFilterBuilder outputFilter = null;
  882. // First off: document discard aliases:
  883. if (null != _aliasLookup) { // Check out the document discard table...
  884. EntityFeaturePojo docDiscardAlias = _aliasLookup.getAliases("DOCUMENT_DISCARD");
  885. if (null != docDiscardAlias) {
  886. outputFilter = addNegativeSelectorToFilter(docDiscardAlias, outputFilter, 0);
  887. }
  888. }//TESTED (by hand, nothing repeatable)
  889. // Other simple filter types:
  890. if (null != entityTypeFilterStrings) {
  891. if ('-' != entityTypeFilterStrings[0].charAt(0)) { // (negative entity type, don't add to filter)
  892. outputFilter = FilterBuilders.boolFilter();
  893. outputFilter.must(FilterBuilders.nestedFilter(DocumentPojo.entities_,
  894. FilterBuilders.termsFilter(EntityPojo.docQuery_type_, entityTypeFilterStrings)));
  895. }
  896. }
  897. if (null != assocVerbFilterStrings) {
  898. if ('-' != assocVerbFilterStrings[0].charAt(0)) { // (negative verb, don't add to filter)
  899. if (null == outputFilter) {
  900. outputFilter = FilterBuilders.boolFilter();
  901. }
  902. BoolFilterBuilder verbFilter = FilterBuilders.boolFilter();
  903. StringBuffer sb = new StringBuffer();
  904. for (String assocVerb: assocVerbFilterStrings) {
  905. sb.setLength(0);
  906. sb.append('"').append(assocVerb).append('"');
  907. verbFilter.should(FilterBuilders.nestedFilter(DocumentPojo.associations_,
  908. QueryBuilders.queryString(sb.toString()).field(AssociationPojo.docQuery_verb_category_)));
  909. //(closest to exact that we can manage, obv verb_cat should actually be not_analyzed)
  910. }
  911. outputFilter.must(verbFilter);
  912. }
  913. }
  914. return outputFilter;
  915. }//TESTED
  916. ////////////////////////////////////////////////////////////////////////
  917. // 1.2] Query term parsing
  918. // (Not needed any more, but kept here for illustrative purposes)
  919. //private static Pattern _luceneExactPattern = Pattern.compile("([\"+~*?:|&(){}\\[\\]\\^\\!\\-\\\\ ])");
  920. private BaseQueryBuilder parseQueryTerm(AdvancedQueryPojo.QueryTermPojo qt, StringBuffer sQueryTerm) {
  921. BaseQueryBuilder term = null;
  922. BoolQueryBuilder boolTerm = null;
  923. sQueryTerm.setLength(0);
  924. sQueryTerm.append('(');
  925. // 1.1] Free text (Lucene)
  926. boolean nonEmpty = false;
  927. if (null != qt.ftext) { // NOTE term building code below depends on this being 1st clause
  928. nonEmpty = true;
  929. if (qt.ftext.startsWith("$cache")) { // currently not supported
  930. throw new RuntimeException("Don't currently support nested cached queries - coming soon.");
  931. }
  932. sQueryTerm.append('(');
  933. if (null != qt.metadataField) {
  934. sQueryTerm.append(qt.metadataField).append(':');
  935. }
  936. sQueryTerm.append(qt.ftext);
  937. sQueryTerm.append(')');
  938. if (null != qt.metadataField) { // Metadata only
  939. term = QueryBuilders.queryString(qt.ftext).field(qt.metadataField);
  940. }
  941. else {
  942. term = QueryBuilders.queryString(qt.ftext).field("_all").field(DocumentPojo.fullText_);
  943. }
  944. if (null == _scoringParams.adjustAggregateSig) { // auto-decide .. if ftext is set and is non-trivial
  945. if (qt.ftext.contains(" ")) {
  946. _scoringParams.adjustAggregateSig = true;
  947. }
  948. }
  949. }//TESTED (logic0)
  950. // 1.2] Exact text
  951. if (null != qt.etext) { // NOTE term building code below depends on this being 2nd clause
  952. nonEmpty = true;
  953. BaseQueryBuilder termQ = null;
  954. if (sQueryTerm.length() > 1) {
  955. sQueryTerm.append(" AND ");
  956. }
  957. if (qt.etext.equals("*")) { // Special case
  958. termQ = QueryBuilders.matchAllQuery();
  959. }
  960. else { // Normal query
  961. if (null != qt.metadataField) { // Metadata only
  962. termQ = CrossVersionQueryBuilders.matchPhraseQuery(qt.metadataField, qt.etext);
  963. }
  964. else { // Normal query
  965. termQ = QueryBuilders.boolQuery().
  966. should(CrossVersionQueryBuilders.matchPhraseQuery("_all", qt.etext)).
  967. should(CrossVersionQueryBuilders.matchPhraseQuery(DocumentPojo.fullText_, qt.etext));
  968. }
  969. }
  970. sQueryTerm.append('(');
  971. if (null != qt.metadataField) {
  972. sQueryTerm.append(qt.metadataField).append(':');
  973. }
  974. sQueryTerm.append('"');
  975. sQueryTerm.append(qt.etext);
  976. sQueryTerm.append("\")");
  977. if (null == term) {
  978. term = termQ;
  979. }
  980. else {
  981. term = (boolTerm = QueryBuilders.boolQuery().must(term).must(termQ));
  982. }
  983. }//TESTED (logic1)
  984. // Here's where it starts getting interesting:
  985. // 1.3] Entity
  986. if ((null != qt.entity) || (null != qt.entityValue) || ((null == qt.assoc) && (null != qt.sentiment))) { // (if no association specified then sentiment applies to entities)
  987. nonEmpty = true;
  988. if (sQueryTerm.length() > 1) {
  989. sQueryTerm.append(" AND ");
  990. }
  991. sQueryTerm.append('(');
  992. BaseQueryBuilder termQ = QueryBuilders.nestedQuery(DocumentPojo.entities_, this.parseEntityTerm(qt, sQueryTerm, EntityPojo.docQuery_index_)).scoreMode("max").boost((float)1.0);
  993. if (null == term) {
  994. term = termQ;
  995. }
  996. else if (null == boolTerm) {
  997. term = (boolTerm = QueryBuilders.boolQuery().must(term).must(termQ));
  998. }
  999. else {
  1000. term = (boolTerm = boolTerm.must(termQ));
  1001. }
  1002. sQueryTerm.append(')');
  1003. }//TESTED: logic2* TOTEST: alias expansion code (logic3)
  1004. // 1.4] Dates
  1005. if (null != qt.time) {
  1006. nonEmpty = true;
  1007. if (sQueryTerm.length() > 1) {
  1008. sQueryTerm.append(" AND ");
  1009. }
  1010. sQueryTerm.append('(');
  1011. BaseQueryBuilder termQ = this.parseDateTerm(qt.time, sQueryTerm);
  1012. if (null == term) {
  1013. term = termQ;
  1014. }
  1015. else if (null == boolTerm) {
  1016. term = (boolTerm = QueryBuilders.boolQuery().must(term).must(termQ));
  1017. }
  1018. else {
  1019. term = (boolTerm = boolTerm.must(termQ));
  1020. }
  1021. sQueryTerm.append(')');
  1022. }//TESTED (logic5-10)
  1023. // 1.5] Geo
  1024. if (null != qt.geo)
  1025. {
  1026. nonEmpty = true;
  1027. if (sQueryTerm.length() > 1)
  1028. {
  1029. sQueryTerm.append(" AND ");
  1030. }
  1031. sQueryTerm.append('(');
  1032. BaseQueryBuilder termQ = this.parseGeoTerm(qt.geo, sQueryTerm, GeoParseField.ALL);
  1033. if (null != termQ)
  1034. {
  1035. if (null == term)
  1036. {
  1037. term = termQ;
  1038. }
  1039. else if (null == boolTerm)
  1040. {
  1041. term = (boolTerm = QueryBuilders.boolQuery().must(term).must(termQ));
  1042. }
  1043. else
  1044. {
  1045. term = (boolTerm = boolTerm.must(termQ));
  1046. }
  1047. }
  1048. sQueryTerm.append(')');
  1049. } // (end geo)
  1050. if (null == qt.assoc) qt.assoc = qt.event;
  1051. //(continue to support the old "event" name for another release)
  1052. if (null != qt.assoc) {
  1053. nonEmpty = true;
  1054. if (sQueryTerm.length() > 1) {
  1055. sQueryTerm.append(" AND ");
  1056. }
  1057. sQueryTerm.append('(');
  1058. BaseQueryBuilder termQ = QueryBuilders.nestedQuery(DocumentPojo.associations_, this.parseAssociationTerm(qt.assoc, qt.sentiment, sQueryTerm));
  1059. if (null != termQ) {
  1060. if (null == term) {
  1061. term = termQ;
  1062. }
  1063. else if (null == boolTerm) {
  1064. term = (boolTerm = QueryBuilders.boolQuery().must(term).must(termQ));
  1065. }
  1066. else {
  1067. term = (boolTerm = boolTerm.must(termQ));
  1068. }
  1069. }
  1070. sQueryTerm.append(')');
  1071. } // (end event)
  1072. if (!nonEmpty) {
  1073. throw new RuntimeException("One+ of your query terms is empty!");
  1074. }//TOTEST
  1075. sQueryTerm.append(')');
  1076. return term;
  1077. }//TESTED (logic*) TOTEST event logic
  1078. //TESTED: each of the above cases with the following GUI commands:
  1079. // infiniteService.send('{"raw": { "match_all": {} } }');
  1080. // infiniteService.send('{ "input": { "tags": [ "healthcare", "cyber" ] } }');
  1081. // infiniteService.send('{ "input": { "typeAndTags": [ { "type": "Social" }, { "type": "Video", "tags": [ "education", "MIT" ] } ] } }');
  1082. // infiniteService.send('{ "input": { "typeAndTags": [ { "type": "Social" }, { "type": "Video", "tags": [ "education…

Large files files are truncated, but you can click here to view the full file