PageRenderTime 23ms CodeModel.GetById 1ms RepoModel.GetById 0ms app.codeStats 0ms

/new_backend/abdelrahmanrustom-searchengine-ac46608ebeab/searches/commands/SearchCommand.java

https://bitbucket.org/abdelrahmanrustom/searchengine
Java | 175 lines | 127 code | 20 blank | 28 comment | 17 complexity | 1e3a9ca6ad78f932080ad4ce1c506cce MD5 | raw file
  1. import com.google.gson.Gson;
  2. import com.google.gson.JsonObject;
  3. import com.mongodb.MongoClient;
  4. import com.mongodb.MongoClientURI;
  5. import com.mongodb.client.MongoCollection;
  6. import com.mongodb.client.MongoDatabase;
  7. import org.apache.http.HttpHost;
  8. import org.bson.Document;
  9. import org.elasticsearch.action.ActionListener;
  10. import org.elasticsearch.action.search.SearchRequest;
  11. import org.elasticsearch.action.search.SearchRequestBuilder;
  12. import org.elasticsearch.action.search.SearchResponse;
  13. import org.elasticsearch.client.RestClient;
  14. import org.elasticsearch.client.RestHighLevelClient;
  15. import org.elasticsearch.common.document.DocumentField;
  16. import org.elasticsearch.index.query.BoolQueryBuilder;
  17. import org.elasticsearch.index.query.QueryBuilder;
  18. import org.elasticsearch.index.query.QueryBuilders;
  19. import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
  20. import org.elasticsearch.search.SearchHit;
  21. import org.elasticsearch.search.SearchHits;
  22. import org.elasticsearch.search.builder.SearchSourceBuilder;
  23. import org.tartarus.snowball.ext.PorterStemmer;
  24. import java.io.IOException;
  25. import java.io.StringReader;
  26. import java.util.*;
  27. import java.util.concurrent.Callable;
  28. //https://github.com/harryaskham/Twitter-L-LDA/blob/master/util/Stopwords.java
  29. import static org.elasticsearch.index.query.QueryBuilders.*;
  30. public class SearchCommand implements Callable<String> {
  31. String q;
  32. int resultType = 0, page_no = 1, category = -1, user_id = -1;
  33. boolean disableSpellingCorrection = false;
  34. PorterStemmer porterStemmer = new PorterStemmer();
  35. final RestHighLevelClient client = new RestHighLevelClient(
  36. RestClient.builder(
  37. new HttpHost("localhost", 9200, "http"),
  38. new HttpHost("localhost", 9201, "http")));
  39. private static final MongoClient mongoClient = new
  40. MongoClient(new MongoClientURI("mongodb://localhost:27017"));
  41. private static final MongoDatabase database = mongoClient.getDatabase("history");
  42. private static final MongoCollection<Document> collection = database.getCollection("user_history");
  43. private static String[] stopwords = {"a", "as", "able", "about", "above", "according", "accordingly", "across", "actually", "after", "afterwards", "again", "against", "aint", "all", "allow", "allows", "almost", "alone", "along", "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another", "any", "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear", "appreciate", "appropriate", "are", "arent", "around", "as", "aside", "ask", "asking", "associated", "at", "available", "away", "awfully", "be", "became", "because", "become", "becomes", "becoming", "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best", "better", "between", "beyond", "both", "brief", "but", "by", "cmon", "cs", "came", "can", "cant", "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come", "comes", "concerning", "consequently", "consider", "considering", "contain", "containing", "contains", "corresponding", "could", "couldnt", "course", "currently", "definitely", "described", "despite", "did", "didnt", "different", "do", "does", "doesnt", "doing", "dont", "done", "down", "downwards", "during", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough", "entirely", "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere", "ex", "exactly", "example", "except", "far", "few", "ff", "fifth", "first", "five", "followed", "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore", "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten", "greetings", "had", "hadnt", "happens", "hardly", "has", "hasnt", "have", "havent", "having", "he", "hes", "hello", "help", "hence", "her", "here", "heres", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit", "however", "i", "id", "ill", "im", "ive", "ie", "if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed", "indicate", "indicated", "indicates", "inner", "insofar", "instead", "into", "inward", "is", "isnt", "it", "itd", "itll", "its", "its", "itself", "just", "keep", "keeps", "kept", "know", "knows", "known", "last", "lately", "later", "latter", "latterly", "least", "less", "lest", "let", "lets", "like", "liked", "likely", "little", "look", "looking", "looks", "ltd", "mainly", "many", "may", "maybe", "me", "mean", "meanwhile", "merely", "might", "more", "moreover", "most", "mostly", "much", "must", "my", "myself", "name", "namely", "nd", "near", "nearly", "necessary", "need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non", "none", "noone", "nor", "normally", "not", "nothing", "novel", "now", "nowhere", "obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on", "once", "one", "ones", "only", "onto", "or", "other", "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall", "own", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible", "presumably", "probably", "provides", "que", "quite", "qv", "rather", "rd", "re", "really", "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "said", "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed", "seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven", "several", "shall", "she", "should", "shouldnt", "since", "six", "so", "some", "somebody", "somehow", "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified", "specify", "specifying", "still", "sub", "such", "sup", "sure", "ts", "take", "taken", "tell", "tends", "th", "than", "thank", "thanks", "thanx", "that", "thats", "thats", "the", "their", "theirs", "them", "themselves", "then", "thence", "there", "theres", "thereafter", "thereby", "therefore", "therein", "theres", "thereupon", "these", "they", "theyd", "theyll", "theyre", "theyve", "think", "third", "this", "thorough", "thoroughly", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries", "truly", "try", "trying", "twice", "two", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon", "us", "use", "used", "useful", "uses", "using", "usually", "value", "various", "very", "via", "viz", "vs", "want", "wants", "was", "wasnt", "way", "we", "wed", "well", "were", "weve", "welcome", "well", "went", "were", "werent", "what", "whats", "whatever", "when", "whence", "whenever", "where", "wheres", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whos", "whoever", "whole", "whom", "whose", "why", "will", "willing", "wish", "with", "within", "without", "wont", "wonder", "would", "would", "wouldnt", "yes", "yet", "you", "youd", "youll", "youre", "youve", "your", "yours", "yourself", "yourselves", "zero"};
  44. private static Set<String> stopWordSet = new HashSet<String>(Arrays.asList(stopwords));
  45. public SearchCommand(String q){
  46. this.q = q;
  47. }
  48. public SearchCommand(String q, int resultType, int page_no, int category,
  49. boolean disableSpellingCorrection, int user_id){
  50. this.q = q;
  51. this.resultType = resultType;
  52. this.page_no = page_no;
  53. this.category = category;
  54. this.disableSpellingCorrection = disableSpellingCorrection;
  55. this.user_id = user_id;
  56. }
  57. private static boolean isStopword(String word) {
  58. if(word.length() < 2) return true;
  59. if(word.charAt(0) >= '0' && word.charAt(0) <= '9') return true; //remove numbers, "25th", etc
  60. if(stopWordSet.contains(word)) return true;
  61. else return false;
  62. }
  63. private String removeStopWordsAndStem(String[] string) {
  64. String result = "";
  65. for(String word : string) {
  66. if(word.isEmpty()) continue;
  67. if(isStopword(word)) continue; //remove stopwords
  68. porterStemmer.setCurrent(word);
  69. porterStemmer.stem();
  70. result += (porterStemmer.getCurrent()+" ");
  71. }
  72. return result;
  73. }
  74. // TODO: max number of pages = 20
  75. public String call(){
  76. final String[] res = {null};
  77. String[] words = q.replaceAll("[^a-zA-Z ]", "").toLowerCase().split("\\s+");;
  78. q = removeStopWordsAndStem(words);
  79. SearchRequest searchRequest = new SearchRequest("webpages");
  80. SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
  81. searchSourceBuilder.from(10*page_no);
  82. searchSourceBuilder.size(10*page_no - 1);
  83. BoolQueryBuilder tmp = QueryBuilders.boolQuery()
  84. .must(QueryBuilders.matchQuery("plain_text",
  85. q))
  86. .should(QueryBuilders.matchQuery("keywords", q));
  87. if(category != -1)
  88. tmp = tmp.must(QueryBuilders.termQuery("category", category));
  89. if(resultType == 1)
  90. tmp = tmp.filter(QueryBuilders.termQuery("has_imgs", true));
  91. if(resultType == 1)
  92. tmp = tmp.filter(QueryBuilders.termQuery("has_vids", true));
  93. searchSourceBuilder.query(tmp);
  94. searchRequest.source(searchSourceBuilder);
  95. Map<String, Object> resJson = new HashMap<>();
  96. if(page_no > 20){
  97. return "{\"error\": \"Maximum number of pages exceeded\"}";
  98. }
  99. if(user_id > 0){
  100. try {
  101. Document d = new Document();
  102. d.put("user_id", user_id);
  103. d.put("query", q);
  104. collection.insertOne(d);
  105. }catch (Exception e){
  106. resJson.put("error", e.toString());
  107. Gson json = new Gson();
  108. res[0] = json.toJson(resJson);
  109. return res[0];
  110. }
  111. }
  112. try {
  113. SearchHits hits = client.search(searchRequest).getHits();
  114. ArrayList<String> json_search_responses = new ArrayList<>();
  115. for(SearchHit hit: hits){
  116. json_search_responses.add(hit.getSourceAsString());
  117. }
  118. // System.out.println(json_search_responses);
  119. resJson.put("results", json_search_responses);
  120. Gson json = new Gson();
  121. res[0] = json.toJson(resJson);
  122. // System.out.println(res[0]);
  123. return res[0];
  124. } catch (IOException e) {
  125. resJson.put("error", e.toString());
  126. Gson json = new Gson();
  127. res[0] = json.toJson(resJson);
  128. return res[0];
  129. }
  130. // client.searchAsync(searchRequest, new ActionListener<SearchResponse>() {
  131. // @Override
  132. // public void onResponse(SearchResponse searchResponse) {
  133. // SearchHits hits = searchResponse.getHits();
  134. // ArrayList<String> json_search_responses = new ArrayList<>();
  135. // for(SearchHit hit: hits){
  136. // json_search_responses.add(hit.getSourceAsString());
  137. // }
  138. //// System.out.println(json_search_responses);
  139. // resJson.put("results", json_search_responses);
  140. // Gson json = new Gson();
  141. // res[0] = json.toJson(resJson);
  142. // System.out.println(res[0]);
  143. // }
  144. //
  145. // @Override
  146. // public void onFailure(Exception e) {
  147. // resJson.put("error", e.toString());
  148. // Gson json = new Gson();
  149. // res[0] = json.toJson(resJson);
  150. // }
  151. // });
  152. // while (!(res[0] == null));
  153. // return res[0];
  154. }
  155. }