PageRenderTime 59ms CodeModel.GetById 32ms RepoModel.GetById 0ms app.codeStats 0ms

/searx/engines/duckduckgo_definitions.py

https://gitlab.com/lanodan/searx
Python | 149 lines | 138 code | 9 blank | 2 comment | 2 complexity | 09cccc9d0b07ac38a3fb0951864321c2 MD5 | raw file
  1. import json
  2. from urllib import urlencode
  3. from lxml import html
  4. from searx.utils import html_to_text
  5. from searx.engines.xpath import extract_text
  6. url = 'https://api.duckduckgo.com/'\
  7. + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
  8. def result_to_text(url, text, htmlResult):
  9. # TODO : remove result ending with "Meaning" or "Category"
  10. dom = html.fromstring(htmlResult)
  11. a = dom.xpath('//a')
  12. if len(a) >= 1:
  13. return extract_text(a[0])
  14. else:
  15. return text
  16. def request(query, params):
  17. # TODO add kl={locale}
  18. params['url'] = url.format(query=urlencode({'q': query}))
  19. return params
  20. def response(resp):
  21. results = []
  22. search_res = json.loads(resp.text)
  23. content = ''
  24. heading = search_res.get('Heading', '')
  25. attributes = []
  26. urls = []
  27. infobox_id = None
  28. relatedTopics = []
  29. # add answer if there is one
  30. answer = search_res.get('Answer', '')
  31. if answer != '':
  32. results.append({'answer': html_to_text(answer)})
  33. # add infobox
  34. if 'Definition' in search_res:
  35. content = content + search_res.get('Definition', '')
  36. if 'Abstract' in search_res:
  37. content = content + search_res.get('Abstract', '')
  38. # image
  39. image = search_res.get('Image', '')
  40. image = None if image == '' else image
  41. # attributes
  42. if 'Infobox' in search_res:
  43. infobox = search_res.get('Infobox', None)
  44. if 'content' in infobox:
  45. for info in infobox.get('content'):
  46. attributes.append({'label': info.get('label'),
  47. 'value': info.get('value')})
  48. # urls
  49. for ddg_result in search_res.get('Results', []):
  50. if 'FirstURL' in ddg_result:
  51. firstURL = ddg_result.get('FirstURL', '')
  52. text = ddg_result.get('Text', '')
  53. urls.append({'title': text, 'url': firstURL})
  54. results.append({'title': heading, 'url': firstURL})
  55. # related topics
  56. for ddg_result in search_res.get('RelatedTopics', []):
  57. if 'FirstURL' in ddg_result:
  58. suggestion = result_to_text(ddg_result.get('FirstURL', None),
  59. ddg_result.get('Text', None),
  60. ddg_result.get('Result', None))
  61. if suggestion != heading:
  62. results.append({'suggestion': suggestion})
  63. elif 'Topics' in ddg_result:
  64. suggestions = []
  65. relatedTopics.append({'name': ddg_result.get('Name', ''),
  66. 'suggestions': suggestions})
  67. for topic_result in ddg_result.get('Topics', []):
  68. suggestion = result_to_text(topic_result.get('FirstURL', None),
  69. topic_result.get('Text', None),
  70. topic_result.get('Result', None))
  71. if suggestion != heading:
  72. suggestions.append(suggestion)
  73. # abstract
  74. abstractURL = search_res.get('AbstractURL', '')
  75. if abstractURL != '':
  76. # add as result ? problem always in english
  77. infobox_id = abstractURL
  78. urls.append({'title': search_res.get('AbstractSource'),
  79. 'url': abstractURL})
  80. # definition
  81. definitionURL = search_res.get('DefinitionURL', '')
  82. if definitionURL != '':
  83. # add as result ? as answer ? problem always in english
  84. infobox_id = definitionURL
  85. urls.append({'title': search_res.get('DefinitionSource'),
  86. 'url': definitionURL})
  87. # entity
  88. entity = search_res.get('Entity', None)
  89. # TODO continent / country / department / location / waterfall /
  90. # mountain range :
  91. # link to map search, get weather, near by locations
  92. # TODO musician : link to music search
  93. # TODO concert tour : ??
  94. # TODO film / actor / television / media franchise :
  95. # links to IMDB / rottentomatoes (or scrap result)
  96. # TODO music : link tu musicbrainz / last.fm
  97. # TODO book : ??
  98. # TODO artist / playwright : ??
  99. # TODO compagny : ??
  100. # TODO software / os : ??
  101. # TODO software engineer : ??
  102. # TODO prepared food : ??
  103. # TODO website : ??
  104. # TODO performing art : ??
  105. # TODO prepared food : ??
  106. # TODO programming language : ??
  107. # TODO file format : ??
  108. if len(heading) > 0:
  109. # TODO get infobox.meta.value where .label='article_title'
  110. if image is None and len(attributes) == 0 and len(urls) == 1 and\
  111. len(relatedTopics) == 0 and len(content) == 0:
  112. results.append({
  113. 'url': urls[0]['url'],
  114. 'title': heading,
  115. 'content': content
  116. })
  117. else:
  118. results.append({
  119. 'infobox': heading,
  120. 'id': infobox_id,
  121. 'entity': entity,
  122. 'content': content,
  123. 'img_src': image,
  124. 'attributes': attributes,
  125. 'urls': urls,
  126. 'relatedTopics': relatedTopics
  127. })
  128. return results