PageRenderTime 80ms CodeModel.GetById 19ms RepoModel.GetById 1ms app.codeStats 0ms

/hackerrankops.py

https://gitlab.com/j000sh/hackerrank-to-git
Python | 200 lines | 142 code | 38 blank | 20 comment | 36 complexity | 7774ace4bdf2bf76d3083ffc3a3df547 MD5 | raw file
  1. """
  2. All HackerRank specific logic
  3. """
  4. import re
  5. from requests import Session
  6. from requests.auth import AuthBase, HTTPBasicAuth
  7. import json
  8. import logging # TODO get rid of these print statements!
  9. from bs4 import BeautifulSoup
  10. HR = 'https://www.hackerrank.com'
  11. HR_REST = HR + '/rest'
  12. CONTESTS = '/contests'
  13. CHALLENGES = '/challenges'
  14. SUBMISSIONS = '/submissions'
  15. SUBMISSIONS_GROUPED = SUBMISSIONS + '/grouped'
  16. # TODO use the request auth parameter
  17. class HRAuth(AuthBase):
  18. def __init__(self, username, password, csrf):
  19. self.username = username
  20. self.password = password
  21. self.csrf = csrf
  22. def __call__(self, r):
  23. r.headers['X-CSRF-TOKEN'] = self.csrf
  24. return r
  25. class HRClient():
  26. def __init__(self, username, password):
  27. self.session = Session()
  28. self.session.hooks['response'].append(addArgsToHook(logAndValidate, getCsrf, session = self.session))
  29. self.login(username, password)
  30. def __enter__(self):
  31. return self
  32. def __exit__(self, *args):
  33. self.logout()
  34. # added dummy timeout argument to not skip CSRF passing
  35. def login(self, username, password):
  36. self.session.get(HR + '/dashboard', timeout = 120)
  37. data = {
  38. 'login': username,
  39. 'password': password,
  40. 'remember_me': False,
  41. }
  42. self.session.post(HR + '/auth/login', json = data, timeout = 120)
  43. # added dummy timeout argument to not skip CSRF passing
  44. def logout(self):
  45. return self.session.delete(HR + '/auth/logout', timeout = 120)
  46. # added dummy timeout argument to not skip CSRF passing
  47. def getUserModel(self):
  48. url = HR_REST + CONTESTS + '/master/hackers/me'
  49. json = {"updated_modal_profiled_data": {"updated": True}}
  50. hooks = {'response': addArgsToHook(logAndValidate, getCsrf, session = self.session)}
  51. return self.session.put(url, json = json, hooks = hooks).json()['model']
  52. # TODO add validation and sanity checks on model counts
  53. def getNewModels(self, models):
  54. if not models:
  55. models = {}
  56. contests = {}
  57. url = HR_REST + '/hackers/me/myrank_contests'
  58. contestSlugs = {'master'} | {c['slug'] for c in self.getModels(url, type = 'recent')}
  59. for slug in contestSlugs:
  60. url = HR_REST + CONTESTS + '/' + slug
  61. # get submission info, not models
  62. submissionIds = {s['id'] for s in self.getModels(url + SUBMISSIONS)}
  63. if slug in models and 'submissions' in models[slug]:
  64. submissionIds -= models[slug]['submissions'].keys()
  65. # break out early if contest is already represented
  66. # TODO break each of these separate processes into separate functions and do sequentially
  67. if not submissionIds:
  68. continue
  69. # TODO is this necessary? does every challenge have an id?
  70. # get challenge info, not models
  71. challengeIds = {c['id'] for c in self.getModels(url + CHALLENGES)}
  72. if slug in models and 'challenges' in models[slug]:
  73. challengeIds -= models[slug]['challenges'].keys()
  74. # uncomment if only want challenge data for challenges attempted or with accompanying submissions
  75. #challengeSlugs = {sub['challenge_slug'] for sub in submissions.values()}
  76. #challengeIds = {sub['challenge_id'] for sub in submissions.values()}
  77. # begin creation of contest
  78. contest = {}
  79. contest['model'] = self.session.get(url).json()['model']
  80. contest['submissions'] = self.getModelsKeyed(url + SUBMISSIONS, submissionIds)
  81. contest['challenges'] = self.getModelsKeyed(url + CHALLENGES, challengeIds)
  82. contests[slug] = contest
  83. return contests
  84. def getModelsKeyed(self, url, ids):
  85. models = {}
  86. total = len(ids)
  87. for curr, i in enumerate(ids):
  88. try:
  89. model = self.session.get(url + '/' + str(i), data = {'remaining': total - curr - 1}).json()['model']
  90. if not model:
  91. continue
  92. except ValueError as e:
  93. model = self.session.get(url + '/' + str(i), data = {'remaining': total - curr - 1}).json()['model']
  94. models[i] = model
  95. return models
  96. # get all models from particular GET request
  97. # NOTE must make two calls because order is sometimes not preserved between requests
  98. def getModels(self, url, **params):
  99. r = self.session.get(url, params = params).json()
  100. count = len(r['models'])
  101. total = r['total']
  102. # return models if all have been acquired
  103. if count >= total:
  104. return r['models']
  105. params['limit'] = total
  106. return self.session.get(url, params = params).json()['models']
  107. def addArgsToHook(*factoryArgs, **factoryKwargs):
  108. def responseHook(response, *requestArgs, **requestKwargs):
  109. for func in factoryArgs:
  110. func(response, **factoryKwargs, **requestKwargs)
  111. return response
  112. return responseHook
  113. def mergeModels(models, newModels):
  114. if not newModels:
  115. return models or {}
  116. if not models:
  117. return newModels or {}
  118. for slug in newModels.keys():
  119. if slug not in models:
  120. models[slug] = newModels[slug]
  121. continue
  122. old = models[slug]
  123. new = newModels[slug]
  124. old['model'] = new['model']
  125. old['challenges'].update(new['challenges'])
  126. old['submissions'].update(new['submissions'])
  127. return models
  128. def sortModels(contests):
  129. models = dict()
  130. for co in contests.values():
  131. models[co['model']['created_at']] = (co, 'co')
  132. for ch in co['challenges'].values():
  133. models[ch['created_at']] = (ch, 'ch')
  134. for s in co['submissions'].values():
  135. models[s['created_at']] = (s, 'sub')
  136. return models
  137. def getCsrf(r, *args, **kwargs):
  138. if not kwargs['timeout']:
  139. return
  140. csrfHtml = BeautifulSoup(r.text, 'html.parser').find(id = 'csrf-token')
  141. if csrfHtml:
  142. csrfHtml = csrfHtml['content']
  143. j = None
  144. csrfJson = None
  145. try:
  146. j = json.loads(r.content)
  147. if 'csrf_token' in j:
  148. csrfJson = j['csrf_token']
  149. elif '_csrf_token' in j:
  150. csrfJson = j['_csrf_token']
  151. except Exception:
  152. pass
  153. csrf = csrfHtml or csrfJson
  154. if 'session' in kwargs and csrf:
  155. kwargs['session'].headers.update({'X-CSRF-TOKEN': csrf})
  156. def logAndValidate(r, *args, **kwargs):
  157. parts = [r.status_code, r.request.method, r.request.url]
  158. if r.request.body:
  159. parts.append(r.request.body)
  160. print(*parts)
  161. if not r.ok:
  162. raise ValueError('Request Failed: ', r.status_code, r.request.url, r.text)