PageRenderTime 51ms CodeModel.GetById 21ms RepoModel.GetById 1ms app.codeStats 0ms

/corehq/ex-submodules/phonelog/utils.py

https://github.com/dimagi/commcare-hq
Python | 311 lines | 272 code | 28 blank | 11 comment | 36 complexity | 3cea60304358b07e46e24a6012c9d841 MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-2.1
  1. from django.conf import settings
  2. from django.db import transaction
  3. from dimagi.utils.logging import notify_exception
  4. from corehq.apps.users.dbaccessors import get_user_id_by_username
  5. from corehq.apps.users.util import cached_user_id_to_username, format_username
  6. from .models import (
  7. DeviceReportEntry,
  8. ForceCloseEntry,
  9. UserEntry,
  10. UserErrorEntry,
  11. )
  12. from .tasks import send_device_log_to_sumologic
  13. def device_users_by_xform(xform_id):
  14. return list(
  15. UserEntry.objects.filter(xform_id__exact=xform_id)
  16. .distinct('username').values_list('username', flat=True)
  17. )
  18. def _force_list(obj_or_list):
  19. return obj_or_list if isinstance(obj_or_list, list) else [obj_or_list]
  20. def _get_log_entries(report, report_slug):
  21. for subreport in report:
  22. # subreport should be {"log": <one or more entry models>}
  23. if isinstance(subreport, dict) and report_slug in subreport:
  24. entry_or_entries = subreport.get(report_slug)
  25. if isinstance(entry_or_entries, list):
  26. for entry in entry_or_entries:
  27. yield entry
  28. else:
  29. yield entry_or_entries
  30. def _get_logs(form, report_name, report_slug):
  31. """
  32. Returns a list of log entries matching report_name.report_slug
  33. These entries are 1-to-1 with the phonelog models (DeviceReportEntry,
  34. UserErrorEntry, UserEntry).
  35. """
  36. report = form.get(report_name, {}) or {}
  37. if isinstance(report, list):
  38. return list(_get_log_entries(report, report_slug))
  39. return _force_list(report.get(report_slug, []))
  40. @transaction.atomic
  41. def process_device_log(domain, xform, force_logs):
  42. _process_user_subreport(xform)
  43. _process_log_subreport(domain, xform)
  44. _process_user_error_subreport(domain, xform)
  45. _process_force_close_subreport(domain, xform)
  46. if force_logs:
  47. clear_device_log_request(domain, xform)
  48. def _process_user_subreport(xform):
  49. if UserEntry.objects.filter(xform_id=xform.form_id).exists():
  50. return
  51. userlogs = _get_logs(xform.form_data, 'user_subreport', 'user')
  52. to_save = []
  53. for i, log in enumerate(userlogs):
  54. to_save.append(UserEntry(
  55. xform_id=xform.form_id,
  56. i=i,
  57. user_id=log["user_id"],
  58. username=log["username"],
  59. sync_token=log["sync_token"],
  60. server_date=xform.received_on
  61. ))
  62. UserEntry.objects.bulk_create(to_save)
  63. def _process_log_subreport(domain, xform):
  64. if DeviceReportEntry.objects.filter(xform_id=xform.form_id).exists():
  65. return
  66. form_data = xform.form_data
  67. logs = _get_logs(form_data, 'log_subreport', 'log')
  68. to_save = []
  69. for i, log in enumerate(logs):
  70. if not log:
  71. continue
  72. logged_in_username, logged_in_user_id = _get_user_info_from_log(domain, log)
  73. to_save.append(DeviceReportEntry(
  74. xform_id=xform.form_id,
  75. i=i,
  76. domain=domain,
  77. type=log["type"],
  78. msg=log["msg"],
  79. # must accept either date or datetime string
  80. date=log["@date"],
  81. server_date=xform.received_on,
  82. app_version=form_data.get('app_version'),
  83. device_id=form_data.get('device_id'),
  84. username=logged_in_username,
  85. user_id=logged_in_user_id,
  86. ))
  87. DeviceReportEntry.objects.bulk_create(to_save)
  88. def _get_user_info_from_log(domain, log):
  89. logged_in_username = None
  90. logged_in_user_id = None
  91. if log["type"] == 'login':
  92. # j2me log = user_id_prefix-username
  93. logged_in_username = log["msg"].split('-')[1]
  94. cc_username = format_username(logged_in_username, domain)
  95. logged_in_user_id = get_user_id_by_username(cc_username)
  96. elif log["type"] == 'user' and log["msg"][:5] == 'login':
  97. # android log = login|username|user_id
  98. msg_split = log["msg"].split('|')
  99. logged_in_username = msg_split[1]
  100. logged_in_user_id = msg_split[2]
  101. return logged_in_username, logged_in_user_id
  102. def _process_user_error_subreport(domain, xform):
  103. if UserErrorEntry.objects.filter(xform_id=xform.form_id).exists():
  104. return
  105. errors = _get_logs(xform.form_data, 'user_error_subreport', 'user_error')
  106. to_save = []
  107. for i, error in enumerate(errors):
  108. # beta versions have 'version', but the name should now be 'app_build'.
  109. # Probably fine to remove after June 2016.
  110. version = error['app_build'] if 'app_build' in error else error['version']
  111. entry = UserErrorEntry(
  112. domain=domain,
  113. xform_id=xform.form_id,
  114. i=i,
  115. app_id=error['app_id'],
  116. version_number=int(version),
  117. date=error["@date"],
  118. server_date=xform.received_on,
  119. user_id=error['user_id'],
  120. expr=error['expr'],
  121. msg=error['msg'],
  122. session=error['session'],
  123. type=error['type'],
  124. context_node=error.get('context_node', ''),
  125. )
  126. to_save.append(entry)
  127. UserErrorEntry.objects.bulk_create(to_save)
  128. def _process_force_close_subreport(domain, xform):
  129. if ForceCloseEntry.objects.filter(xform_id=xform.form_id).exists():
  130. return
  131. force_closures = _get_logs(xform.form_data, 'force_close_subreport', 'force_close')
  132. to_save = []
  133. for force_closure in force_closures:
  134. # There are some testing versions going around with an outdated schema
  135. # This never made it into an official release, but:
  136. # app_id and user_id might be missing
  137. # early versions have 'build_number' - the name should now be 'app_build'
  138. # All of this is probably fine to remove after, say June 2016.
  139. version = (force_closure['app_build'] if 'app_build' in force_closure
  140. else force_closure['build_number'])
  141. entry = ForceCloseEntry(
  142. domain=domain,
  143. xform_id=xform.form_id,
  144. app_id=force_closure.get('app_id'),
  145. version_number=int(version),
  146. date=force_closure["@date"],
  147. server_date=xform.received_on,
  148. user_id=force_closure.get('user_id'),
  149. type=force_closure['type'],
  150. msg=force_closure['msg'],
  151. android_version=force_closure['android_version'],
  152. device_model=force_closure['device_model'],
  153. session_readable=force_closure['session_readable'],
  154. session_serialized=force_closure['session_serialized'],
  155. )
  156. to_save.append(entry)
  157. ForceCloseEntry.objects.bulk_create(to_save)
  158. class SumoLogicLog(object):
  159. """Compiles devicelog data to be sent to sumologic
  160. More info here: https://docs.google.com/document/d/18sSwv2GRGepOIHthC6lxQAh_aUYgDcTou6w9jL2976o/edit
  161. """
  162. def __init__(self, domain, xform):
  163. self.domain = domain
  164. self.xform = xform
  165. def send_data(self, url):
  166. send_device_log_to_sumologic.delay(url, self.log_subreport(), self._get_header('log'))
  167. send_device_log_to_sumologic.delay(url, self.user_error_subreport(), self._get_header('user_error'))
  168. send_device_log_to_sumologic.delay(url, self.force_close_subreport(), self._get_header('force_close'))
  169. def _get_header(self, fmt):
  170. """
  171. https://docs.google.com/document/d/18sSwv2GRGepOIHthC6lxQAh_aUYgDcTou6w9jL2976o/edit#bookmark=id.ao4j7x5tjvt7
  172. """
  173. environment = 'test-env'
  174. if settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS:
  175. environment = 'cas'
  176. if settings.SERVER_ENVIRONMENT == 'india':
  177. environment = 'india'
  178. if settings.SERVER_ENVIRONMENT == 'production':
  179. environment = 'prod'
  180. return {b"X-Sumo-Category": "{env}/{domain}/{fmt}".format(
  181. env=environment,
  182. domain=self.domain,
  183. fmt=fmt,
  184. ).encode('utf-8')}
  185. def _fill_base_template(self, log):
  186. from corehq.apps.receiverwrapper.util import (
  187. get_version_from_appversion_text,
  188. get_commcare_version_from_appversion_text,
  189. )
  190. template = (
  191. "[log_date={log_date}] "
  192. "[log_submission_date={log_submission_date}] "
  193. "[log_type={log_type}] "
  194. "[domain={domain}] "
  195. "[username={username}] "
  196. "[device_id={device_id}] "
  197. "[app_version={app_version}] "
  198. "[cc_version={cc_version}] "
  199. "[msg={msg}]"
  200. )
  201. appversion_text = self.xform.form_data.get('app_version')
  202. return template.format(
  203. log_date=log.get("@date"),
  204. log_submission_date=self.xform.received_on if self.xform.received_on else None,
  205. log_type=log.get("type"),
  206. domain=self.domain,
  207. username=self._get_user_info(log)[0],
  208. device_id=self.xform.form_data.get('device_id'),
  209. app_version=get_version_from_appversion_text(appversion_text),
  210. cc_version=get_commcare_version_from_appversion_text(appversion_text),
  211. msg=log["msg"],
  212. )
  213. def _get_user_info(self, log):
  214. user_subreport = _get_logs(self.xform.form_data, 'user_subreport', 'user')
  215. username, user_id = _get_user_info_from_log(self.domain, log)
  216. if not user_subreport:
  217. return username or '', user_id or ''
  218. else:
  219. # If it's available, use the first user subreport to infer username and user id
  220. if username is None:
  221. username = user_subreport[0].get('username')
  222. if user_id is None:
  223. user_id = user_subreport[0].get('user_id')
  224. return username, user_id
  225. def log_subreport(self):
  226. logs = _get_logs(self.xform.form_data, 'log_subreport', 'log')
  227. return ("\n"
  228. .join([self._fill_base_template(log) for log in logs if log.get('type') != 'forceclose'])
  229. .encode('utf-8'))
  230. def user_error_subreport(self):
  231. logs = _get_logs(self.xform.form_data, 'user_error_subreport', 'user_error')
  232. log_additions_template = " [app_id={app_id}] [user_id={user_id}] [session={session}] [expr={expr}]"
  233. return ("\n".join(
  234. self._fill_base_template(log) + log_additions_template.format(
  235. app_id=log.get('app_id'),
  236. user_id=log.get('user_id'),
  237. session=log.get('session'),
  238. expr=log.get('expr'),
  239. ) for log in logs
  240. ).encode('utf-8'))
  241. def force_close_subreport(self):
  242. logs = _get_logs(self.xform.form_data, 'force_close_subreport', 'force_close')
  243. log_additions_template = (
  244. " [app_id={app_id}] [user_id={user_id}] [session={session}] "
  245. "[device_model={device_model}]"
  246. )
  247. return ("\n".join(
  248. self._fill_base_template(log) + log_additions_template.format(
  249. app_id=log.get('app_id'),
  250. user_id=log.get('user_id'),
  251. session=log.get('session_readable'),
  252. device_model=log.get('device_model'),
  253. ) for log in logs
  254. ).encode('utf-8'))
  255. def clear_device_log_request(domain, xform):
  256. from corehq.apps.ota.models import DeviceLogRequest
  257. user_subreport = _get_logs(xform.form_data, 'user_subreport', 'user')
  258. username = (user_subreport[0].get('username') if user_subreport
  259. else cached_user_id_to_username(xform.user_id))
  260. try:
  261. if not username:
  262. raise DeviceLogRequest.DoesNotExist()
  263. log_request = DeviceLogRequest.objects.get(
  264. domain=domain,
  265. username=username,
  266. )
  267. except DeviceLogRequest.DoesNotExist:
  268. msg = "Forced log submission, but no corresponding request found."
  269. notify_exception(None, msg, details={
  270. 'domain': domain, 'username': username
  271. })
  272. else:
  273. log_request.delete()