PageRenderTime 32ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 0ms

/odk_viewer/tests/test_exports.py

https://github.com/zoulema/formhub
Python | 2024 lines | 1810 code | 95 blank | 119 comment | 38 complexity | 466d32a40eeeccb0facb39ef0aa0f3ab MD5 | raw file
Possible License(s): BSD-3-Clause

Large files files are truncated, but you can click here to view the full file

  1. from sys import stdout
  2. import os
  3. import datetime
  4. import json
  5. import StringIO
  6. import csv
  7. import tempfile
  8. import zipfile
  9. import shutil
  10. from openpyxl import load_workbook
  11. from time import sleep
  12. from pyxform.builder import create_survey_from_xls
  13. from django.conf import settings
  14. from main.tests.test_base import MainTestCase
  15. from django.utils.dateparse import parse_datetime
  16. from django.core.urlresolvers import reverse
  17. from django.core.files.temp import NamedTemporaryFile
  18. from odk_viewer.xls_writer import XlsWriter
  19. from odk_viewer.views import delete_export, export_list, create_export,\
  20. export_progress, export_download
  21. from pyxform import SurveyElementBuilder
  22. from odk_viewer.models import Export, ParsedInstance
  23. from utils.export_tools import generate_export, increment_index_in_filename,\
  24. dict_to_joined_export, ExportBuilder
  25. from odk_logger.models import Instance, XForm
  26. from main.views import delete_data
  27. from utils.logger_tools import inject_instanceid
  28. from django.core.files.storage import get_storage_class
  29. from odk_viewer.pandas_mongo_bridge import NoRecordsFoundError
  30. from odk_viewer.tasks import create_xls_export
  31. from xlrd import open_workbook
  32. from odk_viewer.models.parsed_instance import _encode_for_mongo
  33. from odk_logger.xform_instance_parser import XFormInstanceParser
  34. class TestExportList(MainTestCase):
  35. def setUp(self):
  36. super(TestExportList, self).setUp()
  37. self._publish_transportation_form()
  38. survey = self.surveys[0]
  39. self._make_submission(
  40. os.path.join(
  41. self.this_directory, 'fixtures', 'transportation',
  42. 'instances', survey, survey + '.xml'))
  43. def test_csv_export_list(self):
  44. kwargs = {'username': self.user.username,
  45. 'id_string': self.xform.id_string,
  46. 'export_type': Export.CSV_EXPORT}
  47. # test csv
  48. url = reverse(export_list, kwargs=kwargs)
  49. response = self.client.get(url)
  50. self.assertEqual(response.status_code, 200)
  51. def test_xls_export_list(self):
  52. kwargs = {'username': self.user.username,
  53. 'id_string': self.xform.id_string,
  54. 'export_type': Export.XLS_EXPORT}
  55. url = reverse(export_list, kwargs=kwargs)
  56. response = self.client.get(url)
  57. self.assertEqual(response.status_code, 200)
  58. def test_kml_export_list(self):
  59. kwargs = {'username': self.user.username,
  60. 'id_string': self.xform.id_string,
  61. 'export_type': Export.KML_EXPORT}
  62. url = reverse(export_list, kwargs=kwargs)
  63. response = self.client.get(url)
  64. self.assertEqual(response.status_code, 200)
  65. def test_zip_export_list(self):
  66. kwargs = {'username': self.user.username,
  67. 'id_string': self.xform.id_string,
  68. 'export_type': Export.ZIP_EXPORT}
  69. url = reverse(export_list, kwargs=kwargs)
  70. response = self.client.get(url)
  71. self.assertEqual(response.status_code, 200)
  72. def test_gdoc_export_list(self):
  73. kwargs = {'username': self.user.username,
  74. 'id_string': self.xform.id_string,
  75. 'export_type': Export.GDOC_EXPORT}
  76. url = reverse(export_list, kwargs=kwargs)
  77. response = self.client.get(url)
  78. self.assertEqual(response.status_code, 302)
  79. def test_xsv_zip_export_list(self):
  80. kwargs = {'username': self.user.username,
  81. 'id_string': self.xform.id_string,
  82. 'export_type': Export.CSV_ZIP_EXPORT}
  83. url = reverse(export_list, kwargs=kwargs)
  84. response = self.client.get(url)
  85. self.assertEqual(response.status_code, 200)
  86. class TestDataExportURL(MainTestCase):
  87. def setUp(self):
  88. super(TestDataExportURL, self).setUp()
  89. self._publish_transportation_form()
  90. def _filename_from_disposition(self, content_disposition):
  91. filename_pos = content_disposition.index('filename=')
  92. self.assertTrue(filename_pos != -1)
  93. return content_disposition[filename_pos + len('filename='):]
  94. def test_csv_export_url(self):
  95. self._submit_transport_instance()
  96. url = reverse('csv_export', kwargs={
  97. 'username': self.user.username,
  98. 'id_string': self.xform.id_string,
  99. })
  100. response = self.client.get(url)
  101. headers = dict(response.items())
  102. self.assertEqual(headers['Content-Type'], 'application/csv')
  103. content_disposition = headers['Content-Disposition']
  104. filename = self._filename_from_disposition(content_disposition)
  105. basename, ext = os.path.splitext(filename)
  106. self.assertEqual(ext, '.csv')
  107. def test_csv_export_url_without_records(self):
  108. # csv using the pandas path can throw a NoRecordsFound Exception -
  109. # handle it gracefully
  110. url = reverse('csv_export', kwargs={
  111. 'username': self.user.username,
  112. 'id_string': self.xform.id_string,
  113. })
  114. response = self.client.get(url)
  115. self.assertEqual(response.status_code, 404)
  116. def test_xls_export_url(self):
  117. self._submit_transport_instance()
  118. url = reverse('xls_export', kwargs={
  119. 'username': self.user.username,
  120. 'id_string': self.xform.id_string,
  121. })
  122. response = self.client.get(url)
  123. headers = dict(response.items())
  124. self.assertEqual(headers['Content-Type'],
  125. 'application/vnd.openxmlformats')
  126. content_disposition = headers['Content-Disposition']
  127. filename = self._filename_from_disposition(content_disposition)
  128. basename, ext = os.path.splitext(filename)
  129. self.assertEqual(ext, '.xlsx')
  130. def test_csv_zip_export_url(self):
  131. self._submit_transport_instance()
  132. url = reverse('csv_zip_export', kwargs={
  133. 'username': self.user.username,
  134. 'id_string': self.xform.id_string,
  135. })
  136. response = self.client.get(url)
  137. headers = dict(response.items())
  138. self.assertEqual(headers['Content-Type'], 'application/zip')
  139. content_disposition = headers['Content-Disposition']
  140. filename = self._filename_from_disposition(content_disposition)
  141. basename, ext = os.path.splitext(filename)
  142. self.assertEqual(ext, '.zip')
  143. class TestExports(MainTestCase):
  144. def setUp(self):
  145. super(TestExports, self).setUp()
  146. self._submission_time = parse_datetime('2013-02-18 15:54:01Z')
  147. def test_unique_xls_sheet_name(self):
  148. xls_writer = XlsWriter()
  149. xls_writer.add_sheet('section9_pit_latrine_with_slab_group')
  150. xls_writer.add_sheet('section9_pit_latrine_without_slab_group')
  151. # create a set of sheet names keys
  152. sheet_names_set = set(xls_writer._sheets.keys())
  153. self.assertEqual(len(sheet_names_set), 2)
  154. def test_csv_http_response(self):
  155. self._publish_transportation_form()
  156. survey = self.surveys[0]
  157. self._make_submission(
  158. os.path.join(
  159. self.this_directory, 'fixtures', 'transportation',
  160. 'instances', survey, survey + '.xml'),
  161. forced_submission_time=self._submission_time)
  162. response = self.client.get(reverse('csv_export',
  163. kwargs={
  164. 'username': self.user.username,
  165. 'id_string': self.xform.id_string
  166. }))
  167. self.assertEqual(response.status_code, 200)
  168. test_file_path = os.path.join(os.path.dirname(__file__),
  169. 'fixtures', 'transportation.csv')
  170. content = self._get_response_content(response)
  171. with open(test_file_path, 'r') as test_file:
  172. self.assertEqual(content, test_file.read())
  173. def test_responses_for_empty_exports(self):
  174. self._publish_transportation_form()
  175. # test csv though xls uses the same view
  176. url = reverse('csv_export',
  177. kwargs={
  178. 'username': self.user.username,
  179. 'id_string': self.xform.id_string
  180. }
  181. )
  182. self.response = self.client.get(url)
  183. self.assertEqual(self.response.status_code, 404)
  184. self.assertIn('text/html', self.response['content-type'])
  185. def test_create_export(self):
  186. self._publish_transportation_form_and_submit_instance()
  187. storage = get_storage_class()()
  188. # test xls
  189. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  190. self.xform.id_string)
  191. self.assertTrue(storage.exists(export.filepath))
  192. path, ext = os.path.splitext(export.filename)
  193. self.assertEqual(ext, '.xls')
  194. # test csv
  195. export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
  196. self.xform.id_string)
  197. self.assertTrue(storage.exists(export.filepath))
  198. path, ext = os.path.splitext(export.filename)
  199. self.assertEqual(ext, '.csv')
  200. # test xls with existing export_id
  201. existing_export = Export.objects.create(xform=self.xform,
  202. export_type=Export.XLS_EXPORT)
  203. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  204. self.xform.id_string, existing_export.id)
  205. self.assertEqual(existing_export.id, export.id)
  206. def test_delete_file_on_export_delete(self):
  207. self._publish_transportation_form()
  208. self._submit_transport_instance()
  209. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  210. self.xform.id_string)
  211. storage = get_storage_class()()
  212. self.assertTrue(storage.exists(export.filepath))
  213. # delete export object
  214. export.delete()
  215. self.assertFalse(storage.exists(export.filepath))
  216. def test_graceful_exit_on_export_delete_if_file_doesnt_exist(self):
  217. self._publish_transportation_form()
  218. self._submit_transport_instance()
  219. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  220. self.xform.id_string)
  221. storage = get_storage_class()()
  222. # delete file
  223. storage.delete(export.filepath)
  224. self.assertFalse(storage.exists(export.filepath))
  225. # clear filename, like it would be in an incomplete export
  226. export.filename = None
  227. export.filedir = None
  228. export.save()
  229. # delete export record, which should try to delete file as well
  230. delete_url = reverse(delete_export, kwargs={
  231. 'username': self.user.username,
  232. 'id_string': self.xform.id_string,
  233. 'export_type': 'xls'
  234. })
  235. post_data = {'export_id': export.id}
  236. response = self.client.post(delete_url, post_data)
  237. self.assertEqual(response.status_code, 302)
  238. def test_delete_oldest_export_on_limit(self):
  239. self._publish_transportation_form()
  240. self._submit_transport_instance()
  241. # create first export
  242. first_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  243. self.xform.id_string)
  244. self.assertIsNotNone(first_export.pk)
  245. # create exports that exceed set limit
  246. for i in range(Export.MAX_EXPORTS):
  247. generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  248. self.xform.id_string)
  249. # first export should be deleted
  250. exports = Export.objects.filter(id=first_export.id)
  251. self.assertEqual(len(exports), 0)
  252. def test_create_export_url(self):
  253. self._publish_transportation_form()
  254. self._submit_transport_instance()
  255. num_exports = Export.objects.count()
  256. # create export
  257. create_export_url = reverse(create_export, kwargs={
  258. 'username': self.user.username,
  259. 'id_string': self.xform.id_string,
  260. 'export_type': Export.XLS_EXPORT
  261. })
  262. response = self.client.post(create_export_url)
  263. self.assertEqual(response.status_code, 302)
  264. self.assertEqual(Export.objects.count(), num_exports + 1)
  265. def test_delete_export_url(self):
  266. self._publish_transportation_form()
  267. self._submit_transport_instance()
  268. # create export
  269. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  270. self.xform.id_string)
  271. exports = Export.objects.filter(id=export.id)
  272. self.assertEqual(len(exports), 1)
  273. delete_url = reverse(delete_export, kwargs={
  274. 'username': self.user.username,
  275. 'id_string': self.xform.id_string,
  276. 'export_type': 'xls'
  277. })
  278. post_data = {'export_id': export.id}
  279. response = self.client.post(delete_url, post_data)
  280. self.assertEqual(response.status_code, 302)
  281. exports = Export.objects.filter(id=export.id)
  282. self.assertEqual(len(exports), 0)
  283. def test_export_progress_output(self):
  284. self._publish_transportation_form()
  285. self._submit_transport_instance()
  286. # create exports
  287. for i in range(2):
  288. generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  289. self.xform.id_string)
  290. self.assertEqual(Export.objects.count(), 2)
  291. # progress for multiple exports
  292. progress_url = reverse(export_progress, kwargs={
  293. 'username': self.user.username,
  294. 'id_string': self.xform.id_string,
  295. 'export_type': 'xls'
  296. })
  297. get_data = {'export_ids': [e.id for e in Export.objects.all()]}
  298. response = self.client.get(progress_url, get_data)
  299. content = json.loads(response.content)
  300. self.assertEqual(len(content), 2)
  301. self.assertEqual(sorted(['url', 'export_id', 'complete', 'filename']),
  302. sorted(content[0].keys()))
  303. def test_auto_export_if_none_exists(self):
  304. self._publish_transportation_form()
  305. self._submit_transport_instance()
  306. # get export list url
  307. num_exports = Export.objects.count()
  308. export_list_url = reverse(export_list, kwargs={
  309. 'username': self.user.username,
  310. 'id_string': self.xform.id_string,
  311. 'export_type': Export.XLS_EXPORT
  312. })
  313. response = self.client.get(export_list_url)
  314. self.assertEqual(Export.objects.count(), num_exports + 1)
  315. def test_dont_auto_export_if_exports_exist(self):
  316. self._publish_transportation_form()
  317. self._submit_transport_instance()
  318. # create export
  319. create_export_url = reverse(create_export, kwargs={
  320. 'username': self.user.username,
  321. 'id_string': self.xform.id_string,
  322. 'export_type': Export.XLS_EXPORT
  323. })
  324. response = self.client.post(create_export_url)
  325. num_exports = Export.objects.count()
  326. export_list_url = reverse(export_list, kwargs={
  327. 'username': self.user.username,
  328. 'id_string': self.xform.id_string,
  329. 'export_type': Export.XLS_EXPORT
  330. })
  331. response = self.client.get(export_list_url)
  332. self.assertEqual(Export.objects.count(), num_exports)
  333. def test_last_submission_time_on_export(self):
  334. self._publish_transportation_form()
  335. self._submit_transport_instance()
  336. # create export
  337. xls_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  338. self.xform.id_string)
  339. num_exports = Export.objects.filter(xform=self.xform,
  340. export_type=Export.XLS_EXPORT).count()
  341. # check that our function knows there are no more submissions
  342. self.assertFalse(Export.exports_outdated(xform=self.xform,
  343. export_type=Export.XLS_EXPORT))
  344. sleep(1)
  345. # force new last submission date on xform
  346. last_submission = self.xform.surveys.order_by('-date_created')[0]
  347. last_submission.date_created += datetime.timedelta(hours=1)
  348. last_submission.save()
  349. # check that our function knows data has changed
  350. self.assertTrue(Export.exports_outdated(xform=self.xform,
  351. export_type=Export.XLS_EXPORT))
  352. # check that requesting list url will generate a new export
  353. export_list_url = reverse(export_list, kwargs={
  354. 'username': self.user.username,
  355. 'id_string': self.xform.id_string,
  356. 'export_type': Export.XLS_EXPORT
  357. })
  358. response = self.client.get(export_list_url)
  359. self.assertEqual(Export.objects.filter(xform=self.xform,
  360. export_type=Export.XLS_EXPORT).count(), num_exports + 1)
  361. # make sure another export type causes auto-generation
  362. num_exports = Export.objects.filter(xform=self.xform,
  363. export_type=Export.CSV_EXPORT).count()
  364. export_list_url = reverse(export_list, kwargs={
  365. 'username': self.user.username,
  366. 'id_string': self.xform.id_string,
  367. 'export_type': Export.CSV_EXPORT
  368. })
  369. response = self.client.get(export_list_url)
  370. self.assertEqual(Export.objects.filter(xform=self.xform,
  371. export_type=Export.CSV_EXPORT).count(), num_exports + 1)
  372. def test_last_submission_time_empty(self):
  373. self._publish_transportation_form()
  374. self._submit_transport_instance()
  375. # create export
  376. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  377. self.xform.id_string)
  378. # set time of last submission to None
  379. export.time_of_last_submission = None
  380. export.save()
  381. self.assertTrue(Export.exports_outdated(xform=self.xform,
  382. export_type=Export.XLS_EXPORT))
  383. def test_invalid_export_type(self):
  384. self._publish_transportation_form()
  385. self._submit_transport_instance()
  386. export_list_url = reverse(export_list, kwargs={
  387. 'username': self.user.username,
  388. 'id_string': self.xform.id_string,
  389. 'export_type': 'invalid'
  390. })
  391. response = self.client.get(export_list_url)
  392. self.assertEqual(response.status_code, 400)
  393. # test create url
  394. create_export_url = reverse(create_export, kwargs={
  395. 'username': self.user.username,
  396. 'id_string': self.xform.id_string,
  397. 'export_type': 'invalid'
  398. })
  399. response = self.client.post(create_export_url)
  400. self.assertEqual(response.status_code, 400)
  401. def test_add_index_to_filename(self):
  402. filename = "file_name-123f.txt"
  403. new_filename = increment_index_in_filename(filename)
  404. expected_filename = "file_name-123f-1.txt"
  405. self.assertEqual(new_filename, expected_filename)
  406. # test file that already has an index
  407. filename = "file_name-123.txt"
  408. new_filename = increment_index_in_filename(filename)
  409. expected_filename = "file_name-124.txt"
  410. self.assertEqual(new_filename, expected_filename)
  411. def test_duplicate_export_filename_is_renamed(self):
  412. self._publish_transportation_form()
  413. self._submit_transport_instance()
  414. # create an export object in the db
  415. # TODO: only works if the time we time we generate the basename is exact to the second with the time the 2nd export is created
  416. basename = "%s_%s" % (self.xform.id_string,
  417. datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
  418. filename = basename + ".csv"
  419. export = Export.objects.create(xform=self.xform,
  420. export_type=Export.CSV_EXPORT, filename=filename)
  421. # 2nd export
  422. export_2 = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
  423. self.xform.id_string)
  424. if export.created_on.timetuple() == export_2.created_on.timetuple():
  425. new_filename = increment_index_in_filename(filename)
  426. self.assertEqual(new_filename, export_2.filename)
  427. else:
  428. stdout.write("duplicate export filename test skipped because export times differ.")
  429. def test_export_download_url(self):
  430. self._publish_transportation_form()
  431. self._submit_transport_instance()
  432. export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
  433. self.xform.id_string)
  434. csv_export_url = reverse(export_download, kwargs={
  435. "username": self.user.username,
  436. "id_string": self.xform.id_string,
  437. "export_type": Export.CSV_EXPORT,
  438. "filename": export.filename
  439. })
  440. response = self.client.get(csv_export_url)
  441. self.assertEqual(response.status_code, 200)
  442. # test xls
  443. export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
  444. self.xform.id_string)
  445. xls_export_url = reverse(export_download, kwargs={
  446. "username": self.user.username,
  447. "id_string": self.xform.id_string,
  448. "export_type": Export.XLS_EXPORT,
  449. "filename": export.filename
  450. })
  451. response = self.client.get(xls_export_url)
  452. self.assertEqual(response.status_code, 200)
  453. def test_404_on_export_io_error(self):
  454. """
  455. Test that we return a 404 when the response_with_mimetype_and_name encounters an IOError
  456. """
  457. self._publish_transportation_form()
  458. self._submit_transport_instance()
  459. export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
  460. self.xform.id_string)
  461. export_url = reverse(export_download, kwargs={
  462. "username": self.user.username,
  463. "id_string": self.xform.id_string,
  464. "export_type": Export.CSV_EXPORT,
  465. "filename": export.filename
  466. })
  467. # delete the export
  468. export.delete()
  469. # access the export
  470. response = self.client.get(export_url)
  471. self.assertEqual(response.status_code, 404)
  472. def test_deleted_submission_not_in_export(self):
  473. self._publish_transportation_form()
  474. initial_count = ParsedInstance.query_mongo(
  475. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  476. count=True)[0]['count']
  477. self._submit_transport_instance(0)
  478. self._submit_transport_instance(1)
  479. count = ParsedInstance.query_mongo(
  480. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  481. count=True)[0]['count']
  482. self.assertEqual(count, initial_count+2)
  483. # get id of second submission
  484. instance_id = Instance.objects.filter(
  485. xform=self.xform).order_by('id').reverse()[0].id
  486. delete_url = reverse(
  487. delete_data, kwargs={"username": self.user.username,
  488. "id_string": self.xform.id_string})
  489. params = {'id': instance_id}
  490. self.client.post(delete_url, params)
  491. count = ParsedInstance.query_mongo(
  492. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  493. count=True)[0]['count']
  494. self.assertEqual(count, initial_count + 1)
  495. # create the export
  496. csv_export_url = reverse(
  497. 'csv_export', kwargs={"username": self.user.username,
  498. "id_string":self.xform.id_string})
  499. response = self.client.get(csv_export_url)
  500. self.assertEqual(response.status_code, 200)
  501. f = StringIO.StringIO(self._get_response_content(response))
  502. csv_reader = csv.reader(f)
  503. num_rows = len([row for row in csv_reader])
  504. f.close()
  505. # number of rows == 2 i.e. initial_count + header plus one row
  506. self.assertEqual(num_rows, initial_count + 2)
  507. def test_edited_submissions_in_exports(self):
  508. self._publish_transportation_form()
  509. initial_count = ParsedInstance.query_mongo(
  510. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  511. count=True)[0]['count']
  512. instance_name = 'transport_2011-07-25_19-05-36'
  513. path = os.path.join(
  514. 'main', 'tests', 'fixtures', 'transportation', 'instances_w_uuid',
  515. instance_name, instance_name + '.xml')
  516. self._make_submission(path)
  517. count = ParsedInstance.query_mongo(
  518. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  519. count=True)[0]['count']
  520. self.assertEqual(count, initial_count+1)
  521. instance = Instance.objects.filter(
  522. xform=self.xform).order_by('id').reverse()[0]
  523. # make edited submission - simulating what enketo would return
  524. instance_name = 'transport_2011-07-25_19-05-36-edited'
  525. path = os.path.join(
  526. 'main', 'tests', 'fixtures', 'transportation', 'instances_w_uuid',
  527. instance_name, instance_name + '.xml')
  528. self._make_submission(path)
  529. count = ParsedInstance.query_mongo(
  530. self.user.username, self.xform.id_string, '{}', '[]', '{}',
  531. count=True)[0]['count']
  532. self.assertEqual(count, initial_count+1)
  533. # create the export
  534. csv_export_url = reverse(
  535. 'csv_export', kwargs={"username": self.user.username,
  536. "id_string":self.xform.id_string})
  537. response = self.client.get(csv_export_url)
  538. self.assertEqual(response.status_code, 200)
  539. f = StringIO.StringIO(self._get_response_content(response))
  540. csv_reader = csv.DictReader(f)
  541. data = [row for row in csv_reader]
  542. f.close()
  543. num_rows = len(data)
  544. # number of rows == initial_count + 1
  545. self.assertEqual(num_rows, initial_count + 1)
  546. key ='transport/loop_over_transport_types_frequency/ambulance/frequency_to_referral_facility'
  547. self.assertEqual(data[initial_count][key], "monthly")
  548. def test_export_ids_dont_have_comma_separation(self):
  549. """
  550. It seems using {{ }} to output numbers greater than 1000 formats the
  551. number with a thousand separator
  552. """
  553. self._publish_transportation_form()
  554. self._submit_transport_instance()
  555. # create an in-complete export
  556. export = Export.objects.create(id=1234, xform=self.xform,
  557. export_type=Export.XLS_EXPORT)
  558. self.assertEqual(export.pk, 1234)
  559. export_list_url = reverse(
  560. export_list, kwargs={
  561. "username": self.user.username,
  562. "id_string": self.xform.id_string,
  563. "export_type": Export.XLS_EXPORT
  564. })
  565. response = self.client.get(export_list_url)
  566. self.assertContains(response, '#delete-1234"')
  567. self.assertNotContains(response, '#delete-1,234"')
  568. def test_export_progress_updates(self):
  569. """
  570. Test that after generate_export is called, we change out state to
  571. pending and after its complete, we change it to complete, if we fail
  572. between the two, updates, we have failed
  573. """
  574. self._publish_transportation_form()
  575. # generate an export that fails because of the NoRecordsFound exception
  576. export = Export.objects.create(xform=self.xform,
  577. export_type=Export.XLS_EXPORT)
  578. # check that progress url says pending
  579. progress_url = reverse(export_progress, kwargs={
  580. 'username': self.user.username,
  581. 'id_string': self.xform.id_string,
  582. 'export_type': 'xls'
  583. })
  584. params = {'export_ids': [export.id]}
  585. response = self.client.get(progress_url, params)
  586. status = json.loads(response.content)[0]
  587. self.assertEqual(status["complete"], False)
  588. self.assertEqual(status["filename"], None)
  589. export.internal_status = Export.FAILED
  590. export.save()
  591. # check that progress url says failed
  592. progress_url = reverse(export_progress, kwargs={
  593. 'username': self.user.username,
  594. 'id_string': self.xform.id_string,
  595. 'export_type': 'xls'
  596. })
  597. params = {'export_ids': [export.id]}
  598. response = self.client.get(progress_url, params)
  599. status = json.loads(response.content)[0]
  600. self.assertEqual(status["complete"], True)
  601. self.assertEqual(status["filename"], None)
  602. # make a submission and create a valid export
  603. self._submit_transport_instance()
  604. create_xls_export(
  605. self.user.username,
  606. self.xform.id_string, export.id)
  607. params = {'export_ids': [export.id]}
  608. response = self.client.get(progress_url, params)
  609. status = json.loads(response.content)[0]
  610. self.assertEqual(status["complete"], True)
  611. self.assertIsNotNone(status["filename"])
  612. def test_direct_export_returns_newset_export_if_not_updated_since(self):
  613. self._publish_transportation_form()
  614. self._submit_transport_instance()
  615. self.assertEqual(self.response.status_code, 201)
  616. sleep(1)
  617. self._submit_transport_instance_w_uuid("transport_2011-07-25_19-05-36")
  618. self.assertEqual(self.response.status_code, 201)
  619. initial_num_csv_exports = Export.objects.filter(
  620. xform=self.xform, export_type=Export.CSV_EXPORT).count()
  621. initial_num_xls_exports = Export.objects.filter(
  622. xform=self.xform, export_type=Export.XLS_EXPORT).count()
  623. # request a direct csv export
  624. csv_export_url = reverse('csv_export', kwargs={
  625. 'username': self.user.username,
  626. 'id_string': self.xform.id_string
  627. })
  628. xls_export_url = reverse('xls_export', kwargs={
  629. 'username': self.user.username,
  630. 'id_string': self.xform.id_string
  631. })
  632. response = self.client.get(csv_export_url)
  633. self.assertEqual(response.status_code, 200)
  634. # we should have initial_num_exports + 1 exports
  635. num_csv_exports = Export.objects.filter(
  636. xform=self.xform, export_type=Export.CSV_EXPORT).count()
  637. self.assertEqual(num_csv_exports, initial_num_csv_exports + 1)
  638. # request another export without changing the data
  639. response = self.client.get(csv_export_url)
  640. self.assertEqual(response.status_code, 200)
  641. # we should still only have a single export object
  642. num_csv_exports = Export.objects.filter(
  643. xform=self.xform, export_type=Export.CSV_EXPORT).count()
  644. self.assertEqual(num_csv_exports, initial_num_csv_exports + 1)
  645. # this should not affect a direct XLS export and XLS should still re-generate
  646. response = self.client.get(xls_export_url)
  647. self.assertEqual(response.status_code, 200)
  648. num_xls_exports = Export.objects.filter(
  649. xform=self.xform, export_type=Export.XLS_EXPORT).count()
  650. self.assertEqual(num_xls_exports, initial_num_xls_exports + 1)
  651. # make sure xls doesnt re-generate if data hasn't changed
  652. response = self.client.get(xls_export_url)
  653. self.assertEqual(response.status_code, 200)
  654. num_xls_exports = Export.objects.filter(
  655. xform=self.xform, export_type=Export.XLS_EXPORT).count()
  656. self.assertEqual(num_xls_exports, initial_num_xls_exports + 1)
  657. sleep(1)
  658. # check that data edits cause a re-generation
  659. self._submit_transport_instance_w_uuid(
  660. "transport_2011-07-25_19-05-36-edited")
  661. self.assertEqual(self.response.status_code, 201)
  662. self.client.get(csv_export_url)
  663. self.assertEqual(response.status_code, 200)
  664. # we should have an extra export now that the data has been updated
  665. num_csv_exports = Export.objects.filter(
  666. xform=self.xform, export_type=Export.CSV_EXPORT).count()
  667. self.assertEqual(num_csv_exports, initial_num_csv_exports + 2)
  668. sleep(1)
  669. # and when we delete
  670. delete_url = reverse(delete_data, kwargs={
  671. 'username': self.user.username,
  672. 'id_string': self.xform.id_string
  673. })
  674. instance = Instance.objects.filter().order_by('-pk')[0]
  675. response = self.client.post(delete_url, {'id': instance.id})
  676. self.assertEqual(response.status_code, 200)
  677. response = self.client.get(csv_export_url)
  678. self.assertEqual(response.status_code, 200)
  679. # we should have an extra export now that the data has been updated by the delete
  680. num_csv_exports = Export.objects.filter(
  681. xform=self.xform, export_type=Export.CSV_EXPORT).count()
  682. self.assertEqual(num_csv_exports, initial_num_csv_exports + 3)
  683. def test_exports_outdated_doesnt_consider_failed_exports(self):
  684. self._publish_transportation_form()
  685. self._submit_transport_instance()
  686. # create a bad export
  687. export = Export.objects.create(
  688. xform=self.xform, export_type=Export.XLS_EXPORT,
  689. internal_status=Export.FAILED)
  690. self.assertTrue(
  691. Export.exports_outdated(self.xform, export.export_type))
  692. def test_exports_outdated_considers_pending_exports(self):
  693. self._publish_transportation_form()
  694. self._submit_transport_instance()
  695. # create a pending export
  696. export = Export.objects.create(
  697. xform=self.xform, export_type=Export.XLS_EXPORT,
  698. internal_status=Export.PENDING)
  699. self.assertFalse(
  700. Export.exports_outdated(self.xform, export.export_type))
  701. def _get_csv_data(self, filepath):
  702. storage = get_storage_class()()
  703. csv_file = storage.open(filepath)
  704. reader = csv.DictReader(csv_file)
  705. data = reader.next()
  706. csv_file.close()
  707. return data
  708. def _get_xls_data(self, filepath):
  709. storage = get_storage_class()()
  710. with storage.open(filepath) as f:
  711. workbook = open_workbook(file_contents=f.read())
  712. transportation_sheet = workbook.sheet_by_name("transportation")
  713. self.assertTrue(transportation_sheet.nrows > 1)
  714. headers = transportation_sheet.row_values(0)
  715. column1 = transportation_sheet.row_values(1)
  716. return dict(zip(headers, column1))
  717. def test_column_header_delimiter_export_option(self):
  718. self._publish_transportation_form()
  719. # survey 1 has ambulance and bicycle as values for
  720. # transport/available_transportation_types_to_referral_facility
  721. self._submit_transport_instance(survey_at=1)
  722. create_csv_export_url = reverse(create_export, kwargs={
  723. 'username': self.user.username,
  724. 'id_string': self.xform.id_string,
  725. 'export_type': 'csv'
  726. })
  727. default_params = {}
  728. custom_params = {
  729. 'options[group_delimiter]': '.',
  730. }
  731. # test csv with default group delimiter
  732. response = self.client.post(create_csv_export_url, default_params)
  733. self.assertEqual(response.status_code, 302)
  734. export = Export.objects.filter(
  735. xform=self.xform, export_type='csv').latest('created_on')
  736. self.assertTrue(bool(export.filepath))
  737. data = self._get_csv_data(export.filepath)
  738. self.assertTrue(
  739. data.has_key(
  740. 'transport/available_transportation_types_to_referral_facility/ambulance'))
  741. self.assertEqual(
  742. data['transport/available_transportation_types_to_referral_facility/ambulance'], 'True')
  743. sleep(1)
  744. # test csv with dot delimiter
  745. response = self.client.post(create_csv_export_url, custom_params)
  746. self.assertEqual(response.status_code, 302)
  747. export = Export.objects.filter(
  748. xform=self.xform, export_type='csv').latest('created_on')
  749. self.assertTrue(bool(export.filepath))
  750. data = self._get_csv_data(export.filepath)
  751. self.assertTrue(
  752. data.has_key(
  753. 'transport.available_transportation_types_to_referral_facility.ambulance'))
  754. self.assertEqual(
  755. data['transport.available_transportation_types_to_referral_facility.ambulance'], 'True')
  756. # test xls with default group delimiter
  757. create_csv_export_url = reverse(create_export, kwargs={
  758. 'username': self.user.username,
  759. 'id_string': self.xform.id_string,
  760. 'export_type': 'xls'
  761. })
  762. response = self.client.post(create_csv_export_url, default_params)
  763. self.assertEqual(response.status_code, 302)
  764. export = Export.objects.filter(
  765. xform=self.xform, export_type='xls').latest('created_on')
  766. self.assertTrue(bool(export.filepath))
  767. data = self._get_xls_data(export.filepath)
  768. self.assertTrue(
  769. data.has_key("transport/available_transportation_types_to_referral_facility/ambulance"))
  770. # xlrd reader seems to convert bools into integers i.e. 0 or 1
  771. self.assertEqual(
  772. data["transport/available_transportation_types_to_referral_facility/ambulance"], 1)
  773. sleep(1)
  774. # test xls with dot delimiter
  775. response = self.client.post(create_csv_export_url, custom_params)
  776. self.assertEqual(response.status_code, 302)
  777. export = Export.objects.filter(
  778. xform=self.xform, export_type='xls').latest('created_on')
  779. self.assertTrue(bool(export.filepath))
  780. data = self._get_xls_data(export.filepath)
  781. self.assertTrue(
  782. data.has_key("transport.available_transportation_types_to_referral_facility.ambulance"))
  783. # xlrd reader seems to convert bools into integers i.e. 0 or 1
  784. self.assertEqual(
  785. data["transport.available_transportation_types_to_referral_facility.ambulance"], 1)
  786. def test_split_select_multiple_export_option(self):
  787. self._publish_transportation_form()
  788. self._submit_transport_instance(survey_at=1)
  789. create_csv_export_url = reverse(create_export, kwargs={
  790. 'username': self.user.username,
  791. 'id_string': self.xform.id_string,
  792. 'export_type': 'csv'
  793. })
  794. default_params = {}
  795. custom_params = {
  796. 'options[dont_split_select_multiples]': 'yes'
  797. }
  798. # test csv with default split select multiples
  799. response = self.client.post(create_csv_export_url, default_params)
  800. self.assertEqual(response.status_code, 302)
  801. export = Export.objects.filter(
  802. xform=self.xform, export_type='csv').latest('created_on')
  803. self.assertTrue(bool(export.filepath))
  804. data = self._get_csv_data(export.filepath)
  805. # we should have transport/available_transportation_types_to_referral_facility/ambulance as a separate column
  806. self.assertTrue(
  807. data.has_key(
  808. 'transport/available_transportation_types_to_referral_facility/ambulance'))
  809. sleep(1)
  810. # test csv without default split select multiples
  811. response = self.client.post(create_csv_export_url, custom_params)
  812. self.assertEqual(response.status_code, 302)
  813. export = Export.objects.filter(
  814. xform=self.xform, export_type='csv').latest('created_on')
  815. self.assertTrue(bool(export.filepath))
  816. data = self._get_csv_data(export.filepath)
  817. # transport/available_transportation_types_to_referral_facility/ambulance should not be in its own column
  818. self.assertFalse(
  819. data.has_key(
  820. 'transport/available_transportation_types_to_referral_facility/ambulance'))
  821. # transport/available_transportation_types_to_referral_facility should be a column
  822. self.assertTrue(
  823. data.has_key(
  824. 'transport/available_transportation_types_to_referral_facility'))
  825. # check that ambulance is one the values within the transport/available_transportation_types_to_referral_facility column
  826. self.assertTrue("ambulance" in data['transport/available_transportation_types_to_referral_facility'].split(" "))
  827. create_xls_export_url = reverse(create_export, kwargs={
  828. 'username': self.user.username,
  829. 'id_string': self.xform.id_string,
  830. 'export_type': 'xls'
  831. })
  832. # test xls with default split select multiples
  833. response = self.client.post(create_xls_export_url, default_params)
  834. self.assertEqual(response.status_code, 302)
  835. export = Export.objects.filter(
  836. xform=self.xform, export_type='xls').latest('created_on')
  837. self.assertTrue(bool(export.filepath))
  838. data = self._get_xls_data(export.filepath)
  839. # we should have transport/available_transportation_types_to_referral_facility/ambulance as a separate column
  840. self.assertTrue(
  841. data.has_key(
  842. 'transport/available_transportation_types_to_referral_facility/ambulance'))
  843. sleep(1)
  844. # test xls without default split select multiples
  845. response = self.client.post(create_xls_export_url, custom_params)
  846. self.assertEqual(response.status_code, 302)
  847. export = Export.objects.filter(
  848. xform=self.xform, export_type='xls').latest('created_on')
  849. self.assertTrue(bool(export.filepath))
  850. data = self._get_xls_data(export.filepath)
  851. # transport/available_transportation_types_to_referral_facility/ambulance should NOT be in its own column
  852. self.assertFalse(
  853. data.has_key(
  854. 'transport/available_transportation_types_to_referral_facility/ambulance'))
  855. # transport/available_transportation_types_to_referral_facility should be a column
  856. self.assertTrue(
  857. data.has_key(
  858. 'transport/available_transportation_types_to_referral_facility'))
  859. # check that ambulance is one the values within the transport/available_transportation_types_to_referral_facility column
  860. self.assertTrue("ambulance" in data['transport/available_transportation_types_to_referral_facility'].split(" "))
  861. def test_dict_to_joined_export_works(self):
  862. data =\
  863. {
  864. 'name': 'Abe',
  865. 'age': '35',
  866. '_geolocation': [None, None],
  867. 'attachments': ['abcd.jpg', 'efgh.jpg'],
  868. 'children':
  869. [
  870. {
  871. 'children/name': 'Mike',
  872. 'children/age': '5',
  873. 'children/cartoons':
  874. [
  875. {
  876. 'children/cartoons/name': 'Tom & Jerry',
  877. 'children/cartoons/why': 'Tom is silly',
  878. },
  879. {
  880. 'children/cartoons/name': 'Flinstones',
  881. 'children/cartoons/why': u"I like bamb bam\u0107",
  882. }
  883. ]
  884. },
  885. {
  886. 'children/name': 'John',
  887. 'children/age': '2',
  888. 'children/cartoons':[]
  889. },
  890. {
  891. 'children/name': 'Imora',
  892. 'children/age': '3',
  893. 'children/cartoons':
  894. [
  895. {
  896. 'children/cartoons/name': 'Shrek',
  897. 'children/cartoons/why': 'He\'s so funny'
  898. },
  899. {
  900. 'children/cartoons/name': 'Dexter\'s Lab',
  901. 'children/cartoons/why': 'He thinks hes smart',
  902. 'children/cartoons/characters':
  903. [
  904. {
  905. 'children/cartoons/characters/name': 'Dee Dee',
  906. 'children/cartoons/characters/good_or_evil': 'good'
  907. },
  908. {
  909. 'children/cartoons/characters/name': 'Dexter',
  910. 'children/cartoons/characters/good_or_evil': 'evil'
  911. },
  912. ]
  913. }
  914. ]
  915. }
  916. ]
  917. }
  918. expected_output =\
  919. {
  920. 'survey': {
  921. 'name': 'Abe',
  922. 'age': '35'
  923. },
  924. 'children':
  925. [
  926. {
  927. 'children/name': 'Mike',
  928. 'children/age': '5',
  929. '_index': 1,
  930. '_parent_table_name': 'survey',
  931. '_parent_index': 1
  932. },
  933. {
  934. 'children/name': 'John',
  935. 'children/age': '2',
  936. '_index': 2,
  937. '_parent_table_name': 'survey',
  938. '_parent_index': 1
  939. },
  940. {
  941. 'children/name': 'Imora',
  942. 'children/age': '3',
  943. '_index': 3,
  944. '_parent_table_name': 'survey',
  945. '_parent_index': 1
  946. },
  947. ],
  948. 'children/cartoons':
  949. [
  950. {
  951. 'children/cartoons/name': 'Tom & Jerry',
  952. 'children/cartoons/why': 'Tom is silly',
  953. '_index': 1,
  954. '_parent_table_name': 'children',
  955. '_parent_index': 1
  956. },
  957. {
  958. 'children/cartoons/name': 'Flinstones',
  959. 'children/cartoons/why': u"I like bamb bam\u0107",
  960. '_index': 2,
  961. '_parent_table_name': 'children',
  962. '_parent_index': 1
  963. },
  964. {
  965. 'children/cartoons/name': 'Shrek',
  966. 'children/cartoons/why': 'He\'s so funny',
  967. '_index': 3,
  968. '_parent_table_name': 'children',
  969. '_parent_index': 3
  970. },
  971. {
  972. 'children/cartoons/name': 'Dexter\'s Lab',
  973. 'children/cartoons/why': 'He thinks hes smart',
  974. '_index': 4,
  975. '_parent_table_name': 'children',
  976. '_parent_index': 3
  977. }
  978. ],
  979. 'children/cartoons/characters':
  980. [
  981. {
  982. 'children/cartoons/characters/name': 'Dee Dee',
  983. 'children/cartoons/characters/good_or_evil': 'good',
  984. '_index': 1,
  985. '_parent_table_name': 'children/cartoons',
  986. '_parent_index': 4
  987. },
  988. {
  989. 'children/cartoons/characters/name': 'Dexter',
  990. 'children/cartoons/characters/good_or_evil': 'evil',
  991. '_index': 2,
  992. '_parent_table_name': 'children/cartoons',
  993. '_parent_index': 4
  994. }
  995. ]
  996. }
  997. survey_name = 'survey'
  998. indices = {survey_name: 0}
  999. output = dict_to_joined_export(data, 1, indices, survey_name)
  1000. self.assertEqual(output[survey_name], expected_output[survey_name])
  1001. # 1st level
  1002. self.assertEqual(len(output['children']), 3)
  1003. for child in enumerate(['Mike', 'John', 'Imora']):
  1004. index = child[0]
  1005. name = child[1]
  1006. self.assertEqual(
  1007. filter(
  1008. lambda x: x['children/name'] == name,
  1009. output['children'])[0],
  1010. expected_output['children'][index])
  1011. # 2nd level
  1012. self.assertEqual(len(output['children/cartoons']), 4)
  1013. for cartoon in enumerate(
  1014. ['Tom & Jerry', 'Flinstones', 'Shrek', 'Dexter\'s Lab']):
  1015. index = cartoon[0]
  1016. name = cartoon[1]
  1017. self.assertEqual(
  1018. filter(
  1019. lambda x: x['children/cartoons/name'] == name,
  1020. output['children/cartoons'])[0],
  1021. expected_output['children/cartoons'][index])
  1022. # 3rd level
  1023. self.assertEqual(len(output['children/cartoons/characters']), 2)
  1024. for characters in enumerate(['Dee Dee', 'Dexter']):
  1025. index = characters[0]
  1026. name = characters[1]
  1027. self.assertEqual(
  1028. filter(
  1029. lambda x: x['children/cartoons/characters/name'] == name,
  1030. output['children/cartoons/characters'])[0],
  1031. expected_output['children/cartoons/characters'][index])
  1032. def test_generate_csv_zip_export(self):
  1033. # publish xls form
  1034. self._publish_transportation_form_and_submit_instance()
  1035. # create export db object
  1036. export = generate_export(
  1037. Export.CSV_ZIP_EXPORT, "zip", self.user.username,
  1038. self.xform.id_string, group_delimiter='/',
  1039. split_select_multiples=True)
  1040. storage = get_storage_class()()
  1041. self.assertTrue(storage.exists(export.filepath))
  1042. path, ext = os.path.splitext(export.filename)
  1043. self.assertEqual(ext, '.zip')
  1044. class TestExportBuilder(MainTestCase):
  1045. data = [
  1046. {
  1047. 'name': 'Abe',
  1048. 'age': 35,
  1049. 'tel/telLg==office': '020123456',
  1050. 'children':
  1051. [
  1052. {
  1053. 'children/name': 'Mike',
  1054. 'children/age': 5,
  1055. 'children/fav_colors': 'red blue',
  1056. 'children/iceLg==creams': 'vanilla chocolate',
  1057. 'children/cartoons':
  1058. [
  1059. {
  1060. 'children/cartoons/name': 'Tom & Jerry',

Large files files are truncated, but you can click here to view the full file