PageRenderTime 53ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 0ms

/TwitterAPI/conversation.py

https://gitlab.com/sanchezfauste/TweetDigraph
Python | 108 lines | 90 code | 17 blank | 1 comment | 16 complexity | 97dfd84c456cacc6e20387409eaac3e5 MD5 | raw file
  1. from bs4 import BeautifulSoup
  2. import requests
  3. from Conversation.models import Tweet
  4. from django.utils import timezone
  5. from Vibe.vibe import Vibe
  6. class Conversation:
  7. def __init__(self, root_status_id, tweepy_api, verbose = False, save_on_load = False):
  8. self.root_conv_node = ConversationNode(
  9. tweepy_api.get_status(root_status_id))
  10. self.tweepy_api = tweepy_api
  11. self.verbose = verbose
  12. self.save_on_load = save_on_load
  13. #self.get_status_replies_recursive(self.root_conv_node)
  14. self.get_status_replies_iterative()
  15. def get_status_replies_recursive(self, conv_node):
  16. if self.verbose: conv_node.show()
  17. if self.save_on_load: conv_node.save()
  18. for status_id in conv_node.get_replies_ids():
  19. reply_status = self.tweepy_api.get_status(status_id)
  20. if reply_status.in_reply_to_status_id == conv_node.status.id:
  21. reply_conv_node = conv_node.add_reply(reply_status)
  22. self.get_status_replies_revursive(reply_conv_node)
  23. def get_status_replies_iterative(self):
  24. stack = [self.root_conv_node]
  25. while (len(stack)):
  26. conv_node = stack.pop()
  27. if self.verbose: conv_node.show()
  28. if self.save_on_load: conv_node.save()
  29. for status_id in conv_node.get_replies_ids():
  30. reply_status = self.tweepy_api.get_status(status_id)
  31. if reply_status.in_reply_to_status_id == conv_node.status.id:
  32. reply_conv_node = conv_node.add_reply(reply_status)
  33. stack.append(reply_conv_node)
  34. def show(self):
  35. self.root_conv_node.show_conversation()
  36. def save(self):
  37. self.root_conv_node.save_conversation()
  38. class ConversationNode:
  39. def __init__(self, status, level = 0):
  40. self.status = status
  41. self.replies = []
  42. self.level = level
  43. def add_reply(self, status):
  44. reply = ConversationNode(status, self.level + 1)
  45. self.replies.append(reply)
  46. return reply
  47. def save(self):
  48. tweet = Tweet(id = self.status.id,
  49. username = self.status.user.screen_name,
  50. display_name = self.status.user.name,
  51. profile_image_url = self.status.user.profile_image_url,
  52. created_at = timezone.now(),
  53. text = self.status.text,
  54. retweet_count = self.status.retweet_count,
  55. favourites_count = self.status.favorite_count,
  56. lang = self.status.lang,
  57. sentiment = Vibe(self.status.lang).get_sentiment(self.status.text))
  58. try:
  59. if self.status.in_reply_to_status_id:
  60. tweet.in_reply_to_status_id = Tweet.objects.get(pk=self.status.in_reply_to_status_id)
  61. except:
  62. pass
  63. tweet.save()
  64. def save_conversation(self):
  65. self.save()
  66. for reply in self.replies:
  67. reply.save_conversation()
  68. def show(self):
  69. print " " * self.level + self.status.text
  70. def show_conversation(self):
  71. self.show()
  72. for reply in self.replies:
  73. reply.show_conversation()
  74. def get_replies_ids(self):
  75. url = "https://twitter.com/" + self.status.user.screen_name \
  76. + "/status/" + str(self.status.id)
  77. req = requests.get(url)
  78. if req.status_code == 200:
  79. html = BeautifulSoup(req.text, 'html.parser')
  80. conversations = html.find_all('li', {'class':'ThreadedConversation'})
  81. conversations += html.find_all('div', \
  82. {'class':'ThreadedConversation--loneTweet'})
  83. replies = []
  84. for conversation in conversations:
  85. first_tweet = conversation.find('li', {'class':'js-stream-item ' \
  86. + 'stream-item stream-item\n'})
  87. replies.append(first_tweet.get('data-item-id'))
  88. return replies
  89. else:
  90. print "Error getting status replies ids. HTTP status:", req.status_code
  91. return list()