PageRenderTime 96ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 0ms

/lib/indexer_jobs.rb

https://bitbucket.org/bjfish/indexer_jobs
Ruby | 176 lines | 138 code | 36 blank | 2 comment | 2 complexity | 7f79d31913309e109e22ddf76c20fb73 MD5 | raw file
  1. require "indexer_jobs/version"
  2. require "github_api"
  3. require 'resque/job_with_status'
  4. require 'net/http'
  5. require 'json'
  6. require 'uri'
  7. require 'trello'
  8. require 'tire'
  9. require 'resque'
  10. require 'bitbucket_rest_api'
  11. module IndexerJobs
  12. class GithubJob
  13. include Resque::Plugins::Status
  14. @queue = :github
  15. def perform
  16. Tire.configure do
  17. url ENV['ELASTICSEARCH_URL']
  18. end
  19. at(1, 100, "Indexing started")
  20. token = options['token']
  21. secret = options['secret']
  22. index_id = options['index_id']
  23. github = Github.new oauth_token: token
  24. at(50, 100, "Retrieving documents")
  25. all_issues = github.issues.list
  26. Tire.index index_id do
  27. import all_issues do |documents|
  28. documents.each do |document|
  29. document[:type] = :github_issue
  30. document[:appname] = "Github"
  31. document[:appdocname] = "Issue"
  32. document[:name] = document['title']
  33. document[:id] = document['id']
  34. document[:url] = document['html_url']
  35. end
  36. end
  37. end
  38. at(100, 100, "Indexing Completed")
  39. completed "Indexing completed"
  40. end
  41. def update_document_count(index_id)
  42. uri = URI.parse("#{ENV['ELASTICSEARCH_URL']}/#{index_id}/_count") #TODO Cache this count
  43. response = Net::HTTP.get_response(uri)
  44. hash = JSON.parse response.body
  45. count = hash["count"]
  46. Rails.cache.write({:user_id => index_id, :name => :document_count}, count)
  47. end
  48. end
  49. class BitbucketJob
  50. include Resque::Plugins::Status
  51. @queue = :bitbucket
  52. def perform
  53. es_url = ENV['ELASTICSEARCH_URL']
  54. Tire.configure do
  55. url es_url
  56. end
  57. at(1, 100, "Indexing started")
  58. token = options['token']
  59. secret = options['secret']
  60. index_id = options['index_id']
  61. bitbucket = BitBucket.new do |config|
  62. config.oauth_token = token
  63. config.oauth_secret = secret
  64. config.client_id = ENV['BITBUCKET_API_KEY']
  65. config.client_secret = ENV['BITBUCKET_API_SECRET']
  66. config.adapter = :net_http
  67. end
  68. at(50, 100, "Retrieving documents")
  69. all_issues = []
  70. bitbucket.repos.list do |repo|
  71. if repo.has_issues
  72. issues = bitbucket.issues.list_repo repo.owner, repo.slug
  73. all_issues += issues
  74. end
  75. end
  76. Tire.index index_id do
  77. create
  78. import all_issues do |documents|
  79. documents.each do |document|
  80. document[:type] = :bitbucket_issue
  81. document[:appname] = "Bitbucket"
  82. document[:appdocname] = "Issue"
  83. document[:name] = document['title']
  84. document[:id] = document[:resource_uri]
  85. document[:url] = "https://bitbucket.org" + document[:resource_uri][17..-1]
  86. end
  87. end
  88. end
  89. at(100, 100, "Indexing Completed")
  90. completed "Indexing completed"
  91. end
  92. end
  93. class TrelloJob
  94. #extend Resque::Plugins::Logger
  95. include Resque::Plugins::Status
  96. include Trello
  97. include Trello::Authorization
  98. @queue = :trello
  99. def perform
  100. es_url = ENV['ELASTICSEARCH_URL']
  101. at(1, 100, "Indexing started")
  102. token = options['token']
  103. index_id = options['index_id']
  104. key = ENV['TRELLO_API_KEY']
  105. secret = ENV['TRELLO_API_SECRET']
  106. params = [token, index_id, key, secret, es_url]
  107. raise "Missing parameter" if params.include?(nil)
  108. Tire.configure do
  109. url es_url# TODO FIX ME NOW
  110. #logger 'elasticsearch.log', :level => 'debug'
  111. end
  112. Trello::Authorization.const_set :AuthPolicy, OAuthPolicy
  113. OAuthPolicy.consumer_credential = OAuthCredential.new key, secret
  114. OAuthPolicy.token = OAuthCredential.new token, nil
  115. me = Member.find("me")
  116. boards = me.boards(:filter => :all)
  117. all_cards = []
  118. at(50, 100, "Retrieving documents")
  119. boards.each do |board|
  120. cards = board.cards
  121. cards.each do |card|
  122. all_cards << card.attributes
  123. end
  124. end
  125. index = Tire.index(index_id)
  126. index.create unless index.exists?
  127. Tire.index index_id do
  128. import all_cards do |documents|
  129. documents.each do |document|
  130. document[:type] = :trello_card
  131. document[:appname] = "Trello"
  132. document[:appdocname] = "Card"
  133. end
  134. end
  135. end
  136. at(99, 100, "Indexing Completed")
  137. completed "Indexing completed"
  138. end
  139. end
  140. end