PageRenderTime 85ms CodeModel.GetById 3ms app.highlight 70ms RepoModel.GetById 1ms app.codeStats 1ms

/odk_viewer/tests/test_exports.py

https://github.com/zoulema/formhub
Python | 2024 lines | 1810 code | 95 blank | 119 comment | 56 complexity | 466d32a40eeeccb0facb39ef0aa0f3ab MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1from sys import stdout
   2import os
   3import datetime
   4import json
   5import StringIO
   6import csv
   7import tempfile
   8import zipfile
   9import shutil
  10from openpyxl import load_workbook
  11from time import sleep
  12from pyxform.builder import create_survey_from_xls
  13from django.conf import settings
  14from main.tests.test_base import MainTestCase
  15from django.utils.dateparse import parse_datetime
  16from django.core.urlresolvers import reverse
  17from django.core.files.temp import NamedTemporaryFile
  18from odk_viewer.xls_writer import XlsWriter
  19from odk_viewer.views import delete_export, export_list, create_export,\
  20    export_progress, export_download
  21from pyxform import SurveyElementBuilder
  22from odk_viewer.models import Export, ParsedInstance
  23from utils.export_tools import generate_export, increment_index_in_filename,\
  24    dict_to_joined_export, ExportBuilder
  25from odk_logger.models import Instance, XForm
  26from main.views import delete_data
  27from utils.logger_tools import inject_instanceid
  28from django.core.files.storage import get_storage_class
  29from odk_viewer.pandas_mongo_bridge import NoRecordsFoundError
  30from odk_viewer.tasks import create_xls_export
  31from xlrd import open_workbook
  32from odk_viewer.models.parsed_instance import _encode_for_mongo
  33from odk_logger.xform_instance_parser import XFormInstanceParser
  34
  35
  36class TestExportList(MainTestCase):
  37    def setUp(self):
  38        super(TestExportList, self).setUp()
  39        self._publish_transportation_form()
  40        survey = self.surveys[0]
  41        self._make_submission(
  42            os.path.join(
  43                self.this_directory, 'fixtures', 'transportation',
  44                'instances', survey, survey + '.xml'))
  45
  46    def test_csv_export_list(self):
  47        kwargs = {'username': self.user.username,
  48                  'id_string': self.xform.id_string,
  49                  'export_type': Export.CSV_EXPORT}
  50
  51        # test csv
  52        url = reverse(export_list, kwargs=kwargs)
  53        response = self.client.get(url)
  54        self.assertEqual(response.status_code, 200)
  55
  56    def test_xls_export_list(self):
  57        kwargs = {'username': self.user.username,
  58                  'id_string': self.xform.id_string,
  59                  'export_type': Export.XLS_EXPORT}
  60        url = reverse(export_list, kwargs=kwargs)
  61        response = self.client.get(url)
  62        self.assertEqual(response.status_code, 200)
  63
  64    def test_kml_export_list(self):
  65        kwargs = {'username': self.user.username,
  66                  'id_string': self.xform.id_string,
  67                  'export_type': Export.KML_EXPORT}
  68        url = reverse(export_list, kwargs=kwargs)
  69        response = self.client.get(url)
  70        self.assertEqual(response.status_code, 200)
  71
  72    def test_zip_export_list(self):
  73        kwargs = {'username': self.user.username,
  74                  'id_string': self.xform.id_string,
  75                  'export_type': Export.ZIP_EXPORT}
  76        url = reverse(export_list, kwargs=kwargs)
  77        response = self.client.get(url)
  78        self.assertEqual(response.status_code, 200)
  79
  80    def test_gdoc_export_list(self):
  81        kwargs = {'username': self.user.username,
  82                  'id_string': self.xform.id_string,
  83                  'export_type': Export.GDOC_EXPORT}
  84        url = reverse(export_list, kwargs=kwargs)
  85        response = self.client.get(url)
  86        self.assertEqual(response.status_code, 302)
  87
  88    def test_xsv_zip_export_list(self):
  89        kwargs = {'username': self.user.username,
  90                  'id_string': self.xform.id_string,
  91                  'export_type': Export.CSV_ZIP_EXPORT}
  92        url = reverse(export_list, kwargs=kwargs)
  93        response = self.client.get(url)
  94        self.assertEqual(response.status_code, 200)
  95
  96
  97class TestDataExportURL(MainTestCase):
  98    def setUp(self):
  99        super(TestDataExportURL, self).setUp()
 100        self._publish_transportation_form()
 101
 102    def _filename_from_disposition(self, content_disposition):
 103        filename_pos = content_disposition.index('filename=')
 104        self.assertTrue(filename_pos != -1)
 105        return content_disposition[filename_pos + len('filename='):]
 106
 107    def test_csv_export_url(self):
 108        self._submit_transport_instance()
 109        url = reverse('csv_export', kwargs={
 110            'username': self.user.username,
 111            'id_string': self.xform.id_string,
 112        })
 113        response = self.client.get(url)
 114        headers = dict(response.items())
 115        self.assertEqual(headers['Content-Type'], 'application/csv')
 116        content_disposition = headers['Content-Disposition']
 117        filename = self._filename_from_disposition(content_disposition)
 118        basename, ext = os.path.splitext(filename)
 119        self.assertEqual(ext, '.csv')
 120
 121    def test_csv_export_url_without_records(self):
 122        # csv using the pandas path can throw a NoRecordsFound Exception -
 123        # handle it gracefully
 124        url = reverse('csv_export', kwargs={
 125            'username': self.user.username,
 126            'id_string': self.xform.id_string,
 127        })
 128        response = self.client.get(url)
 129        self.assertEqual(response.status_code, 404)
 130
 131    def test_xls_export_url(self):
 132        self._submit_transport_instance()
 133        url = reverse('xls_export', kwargs={
 134            'username': self.user.username,
 135            'id_string': self.xform.id_string,
 136        })
 137        response = self.client.get(url)
 138        headers = dict(response.items())
 139        self.assertEqual(headers['Content-Type'],
 140                         'application/vnd.openxmlformats')
 141        content_disposition = headers['Content-Disposition']
 142        filename = self._filename_from_disposition(content_disposition)
 143        basename, ext = os.path.splitext(filename)
 144        self.assertEqual(ext, '.xlsx')
 145
 146    def test_csv_zip_export_url(self):
 147        self._submit_transport_instance()
 148        url = reverse('csv_zip_export', kwargs={
 149            'username': self.user.username,
 150            'id_string': self.xform.id_string,
 151        })
 152        response = self.client.get(url)
 153        headers = dict(response.items())
 154        self.assertEqual(headers['Content-Type'], 'application/zip')
 155        content_disposition = headers['Content-Disposition']
 156        filename = self._filename_from_disposition(content_disposition)
 157        basename, ext = os.path.splitext(filename)
 158        self.assertEqual(ext, '.zip')
 159
 160
 161class TestExports(MainTestCase):
 162    def setUp(self):
 163        super(TestExports, self).setUp()
 164        self._submission_time = parse_datetime('2013-02-18 15:54:01Z')
 165
 166    def test_unique_xls_sheet_name(self):
 167        xls_writer = XlsWriter()
 168        xls_writer.add_sheet('section9_pit_latrine_with_slab_group')
 169        xls_writer.add_sheet('section9_pit_latrine_without_slab_group')
 170        # create a set of sheet names keys
 171        sheet_names_set = set(xls_writer._sheets.keys())
 172        self.assertEqual(len(sheet_names_set), 2)
 173
 174    def test_csv_http_response(self):
 175        self._publish_transportation_form()
 176        survey = self.surveys[0]
 177        self._make_submission(
 178            os.path.join(
 179                self.this_directory, 'fixtures', 'transportation',
 180                'instances', survey, survey + '.xml'),
 181            forced_submission_time=self._submission_time)
 182        response = self.client.get(reverse('csv_export',
 183            kwargs={
 184                'username': self.user.username,
 185                'id_string': self.xform.id_string
 186            }))
 187        self.assertEqual(response.status_code, 200)
 188        test_file_path = os.path.join(os.path.dirname(__file__),
 189            'fixtures', 'transportation.csv')
 190        content = self._get_response_content(response)
 191        with open(test_file_path, 'r') as test_file:
 192            self.assertEqual(content, test_file.read())
 193
 194    def test_responses_for_empty_exports(self):
 195        self._publish_transportation_form()
 196        # test csv though xls uses the same view
 197        url = reverse('csv_export',
 198            kwargs={
 199                'username': self.user.username,
 200                'id_string': self.xform.id_string
 201            }
 202        )
 203        self.response = self.client.get(url)
 204        self.assertEqual(self.response.status_code, 404)
 205        self.assertIn('text/html', self.response['content-type'])
 206
 207    def test_create_export(self):
 208        self._publish_transportation_form_and_submit_instance()
 209        storage = get_storage_class()()
 210        # test xls
 211        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 212            self.xform.id_string)
 213        self.assertTrue(storage.exists(export.filepath))
 214        path, ext = os.path.splitext(export.filename)
 215        self.assertEqual(ext, '.xls')
 216
 217        # test csv
 218        export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
 219            self.xform.id_string)
 220        self.assertTrue(storage.exists(export.filepath))
 221        path, ext = os.path.splitext(export.filename)
 222        self.assertEqual(ext, '.csv')
 223
 224        # test xls with existing export_id
 225        existing_export = Export.objects.create(xform=self.xform,
 226            export_type=Export.XLS_EXPORT)
 227        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 228            self.xform.id_string, existing_export.id)
 229        self.assertEqual(existing_export.id, export.id)
 230
 231    def test_delete_file_on_export_delete(self):
 232        self._publish_transportation_form()
 233        self._submit_transport_instance()
 234        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 235                                 self.xform.id_string)
 236        storage = get_storage_class()()
 237        self.assertTrue(storage.exists(export.filepath))
 238        # delete export object
 239        export.delete()
 240        self.assertFalse(storage.exists(export.filepath))
 241
 242    def test_graceful_exit_on_export_delete_if_file_doesnt_exist(self):
 243        self._publish_transportation_form()
 244        self._submit_transport_instance()
 245        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 246            self.xform.id_string)
 247        storage = get_storage_class()()
 248        # delete file
 249        storage.delete(export.filepath)
 250        self.assertFalse(storage.exists(export.filepath))
 251        # clear filename, like it would be in an incomplete export
 252        export.filename = None
 253        export.filedir = None
 254        export.save()
 255        # delete export record, which should try to delete file as well
 256        delete_url = reverse(delete_export, kwargs={
 257            'username': self.user.username,
 258            'id_string': self.xform.id_string,
 259            'export_type': 'xls'
 260        })
 261        post_data = {'export_id': export.id}
 262        response = self.client.post(delete_url, post_data)
 263        self.assertEqual(response.status_code, 302)
 264
 265    def test_delete_oldest_export_on_limit(self):
 266        self._publish_transportation_form()
 267        self._submit_transport_instance()
 268        # create first export
 269        first_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 270            self.xform.id_string)
 271        self.assertIsNotNone(first_export.pk)
 272        # create exports that exceed set limit
 273        for i in range(Export.MAX_EXPORTS):
 274            generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 275                self.xform.id_string)
 276        # first export should be deleted
 277        exports = Export.objects.filter(id=first_export.id)
 278        self.assertEqual(len(exports), 0)
 279
 280    def test_create_export_url(self):
 281        self._publish_transportation_form()
 282        self._submit_transport_instance()
 283        num_exports = Export.objects.count()
 284        # create export
 285        create_export_url = reverse(create_export, kwargs={
 286            'username': self.user.username,
 287            'id_string': self.xform.id_string,
 288            'export_type': Export.XLS_EXPORT
 289        })
 290        response = self.client.post(create_export_url)
 291        self.assertEqual(response.status_code, 302)
 292        self.assertEqual(Export.objects.count(), num_exports + 1)
 293
 294    def test_delete_export_url(self):
 295        self._publish_transportation_form()
 296        self._submit_transport_instance()
 297        # create export
 298        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 299            self.xform.id_string)
 300        exports = Export.objects.filter(id=export.id)
 301        self.assertEqual(len(exports), 1)
 302        delete_url = reverse(delete_export, kwargs={
 303            'username': self.user.username,
 304            'id_string': self.xform.id_string,
 305            'export_type': 'xls'
 306        })
 307        post_data = {'export_id': export.id}
 308        response = self.client.post(delete_url, post_data)
 309        self.assertEqual(response.status_code, 302)
 310        exports = Export.objects.filter(id=export.id)
 311        self.assertEqual(len(exports), 0)
 312
 313    def test_export_progress_output(self):
 314        self._publish_transportation_form()
 315        self._submit_transport_instance()
 316        # create exports
 317        for i in range(2):
 318            generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 319                self.xform.id_string)
 320        self.assertEqual(Export.objects.count(), 2)
 321        # progress for multiple exports
 322        progress_url = reverse(export_progress, kwargs={
 323            'username': self.user.username,
 324            'id_string': self.xform.id_string,
 325            'export_type': 'xls'
 326        })
 327        get_data = {'export_ids': [e.id for e in Export.objects.all()]}
 328        response = self.client.get(progress_url, get_data)
 329        content = json.loads(response.content)
 330        self.assertEqual(len(content), 2)
 331        self.assertEqual(sorted(['url', 'export_id', 'complete', 'filename']),
 332            sorted(content[0].keys()))
 333
 334    def test_auto_export_if_none_exists(self):
 335        self._publish_transportation_form()
 336        self._submit_transport_instance()
 337        # get export list url
 338        num_exports = Export.objects.count()
 339        export_list_url = reverse(export_list, kwargs={
 340            'username': self.user.username,
 341            'id_string': self.xform.id_string,
 342            'export_type': Export.XLS_EXPORT
 343        })
 344        response = self.client.get(export_list_url)
 345        self.assertEqual(Export.objects.count(), num_exports + 1)
 346
 347    def test_dont_auto_export_if_exports_exist(self):
 348        self._publish_transportation_form()
 349        self._submit_transport_instance()
 350        # create export
 351        create_export_url = reverse(create_export, kwargs={
 352            'username': self.user.username,
 353            'id_string': self.xform.id_string,
 354            'export_type': Export.XLS_EXPORT
 355        })
 356        response = self.client.post(create_export_url)
 357        num_exports = Export.objects.count()
 358        export_list_url = reverse(export_list, kwargs={
 359            'username': self.user.username,
 360            'id_string': self.xform.id_string,
 361            'export_type': Export.XLS_EXPORT
 362        })
 363        response = self.client.get(export_list_url)
 364        self.assertEqual(Export.objects.count(), num_exports)
 365
 366    def test_last_submission_time_on_export(self):
 367        self._publish_transportation_form()
 368        self._submit_transport_instance()
 369        # create export
 370        xls_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 371            self.xform.id_string)
 372        num_exports = Export.objects.filter(xform=self.xform,
 373            export_type=Export.XLS_EXPORT).count()
 374        # check that our function knows there are no more submissions
 375        self.assertFalse(Export.exports_outdated(xform=self.xform,
 376            export_type=Export.XLS_EXPORT))
 377        sleep(1)
 378        # force new  last submission date on xform
 379        last_submission = self.xform.surveys.order_by('-date_created')[0]
 380        last_submission.date_created += datetime.timedelta(hours=1)
 381        last_submission.save()
 382        # check that our function knows data has changed
 383        self.assertTrue(Export.exports_outdated(xform=self.xform,
 384            export_type=Export.XLS_EXPORT))
 385        # check that requesting list url will generate a new export
 386        export_list_url = reverse(export_list, kwargs={
 387            'username': self.user.username,
 388            'id_string': self.xform.id_string,
 389            'export_type': Export.XLS_EXPORT
 390        })
 391        response = self.client.get(export_list_url)
 392        self.assertEqual(Export.objects.filter(xform=self.xform,
 393            export_type=Export.XLS_EXPORT).count(), num_exports + 1)
 394        # make sure another export type causes auto-generation
 395        num_exports = Export.objects.filter(xform=self.xform,
 396            export_type=Export.CSV_EXPORT).count()
 397        export_list_url = reverse(export_list, kwargs={
 398            'username': self.user.username,
 399            'id_string': self.xform.id_string,
 400            'export_type': Export.CSV_EXPORT
 401        })
 402        response = self.client.get(export_list_url)
 403        self.assertEqual(Export.objects.filter(xform=self.xform,
 404            export_type=Export.CSV_EXPORT).count(), num_exports + 1)
 405
 406    def test_last_submission_time_empty(self):
 407        self._publish_transportation_form()
 408        self._submit_transport_instance()
 409        # create export
 410        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 411            self.xform.id_string)
 412        # set time of last submission to None
 413        export.time_of_last_submission = None
 414        export.save()
 415        self.assertTrue(Export.exports_outdated(xform=self.xform,
 416            export_type=Export.XLS_EXPORT))
 417
 418    def test_invalid_export_type(self):
 419        self._publish_transportation_form()
 420        self._submit_transport_instance()
 421        export_list_url = reverse(export_list, kwargs={
 422            'username': self.user.username,
 423            'id_string': self.xform.id_string,
 424            'export_type': 'invalid'
 425        })
 426        response = self.client.get(export_list_url)
 427        self.assertEqual(response.status_code, 400)
 428        # test create url
 429        create_export_url = reverse(create_export, kwargs={
 430            'username': self.user.username,
 431            'id_string': self.xform.id_string,
 432            'export_type': 'invalid'
 433        })
 434        response = self.client.post(create_export_url)
 435        self.assertEqual(response.status_code, 400)
 436
 437    def test_add_index_to_filename(self):
 438        filename = "file_name-123f.txt"
 439        new_filename = increment_index_in_filename(filename)
 440        expected_filename = "file_name-123f-1.txt"
 441        self.assertEqual(new_filename, expected_filename)
 442
 443        # test file that already has an index
 444        filename = "file_name-123.txt"
 445        new_filename = increment_index_in_filename(filename)
 446        expected_filename = "file_name-124.txt"
 447        self.assertEqual(new_filename, expected_filename)
 448
 449    def test_duplicate_export_filename_is_renamed(self):
 450        self._publish_transportation_form()
 451        self._submit_transport_instance()
 452        # create an export object in the db
 453        # TODO: only works if the time we time we generate the basename is exact to the second with the time the 2nd export is created
 454        basename = "%s_%s" % (self.xform.id_string,
 455                              datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
 456        filename = basename + ".csv"
 457        export = Export.objects.create(xform=self.xform,
 458            export_type=Export.CSV_EXPORT, filename=filename)
 459        # 2nd export
 460        export_2 = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
 461                                   self.xform.id_string)
 462        if export.created_on.timetuple() == export_2.created_on.timetuple():
 463            new_filename = increment_index_in_filename(filename)
 464            self.assertEqual(new_filename, export_2.filename)
 465        else:
 466            stdout.write("duplicate export filename test skipped because export times differ.")
 467
 468    def test_export_download_url(self):
 469        self._publish_transportation_form()
 470        self._submit_transport_instance()
 471        export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
 472                                 self.xform.id_string)
 473        csv_export_url = reverse(export_download, kwargs={
 474            "username": self.user.username,
 475            "id_string": self.xform.id_string,
 476            "export_type": Export.CSV_EXPORT,
 477            "filename": export.filename
 478        })
 479        response = self.client.get(csv_export_url)
 480        self.assertEqual(response.status_code, 200)
 481        # test xls
 482        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
 483            self.xform.id_string)
 484        xls_export_url = reverse(export_download, kwargs={
 485            "username": self.user.username,
 486            "id_string": self.xform.id_string,
 487            "export_type": Export.XLS_EXPORT,
 488            "filename": export.filename
 489        })
 490        response = self.client.get(xls_export_url)
 491        self.assertEqual(response.status_code, 200)
 492
 493    def test_404_on_export_io_error(self):
 494        """
 495        Test that we return a 404 when the response_with_mimetype_and_name encounters an IOError
 496        """
 497        self._publish_transportation_form()
 498        self._submit_transport_instance()
 499        export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
 500                                 self.xform.id_string)
 501        export_url = reverse(export_download, kwargs={
 502            "username": self.user.username,
 503            "id_string": self.xform.id_string,
 504            "export_type": Export.CSV_EXPORT,
 505            "filename": export.filename
 506        })
 507        # delete the export
 508        export.delete()
 509        # access the export
 510        response = self.client.get(export_url)
 511        self.assertEqual(response.status_code, 404)
 512
 513    def test_deleted_submission_not_in_export(self):
 514        self._publish_transportation_form()
 515        initial_count = ParsedInstance.query_mongo(
 516            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 517            count=True)[0]['count']
 518        self._submit_transport_instance(0)
 519        self._submit_transport_instance(1)
 520        count = ParsedInstance.query_mongo(
 521            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 522            count=True)[0]['count']
 523        self.assertEqual(count, initial_count+2)
 524        # get id of second submission
 525        instance_id = Instance.objects.filter(
 526            xform=self.xform).order_by('id').reverse()[0].id
 527        delete_url = reverse(
 528            delete_data, kwargs={"username": self.user.username,
 529                                 "id_string": self.xform.id_string})
 530        params = {'id': instance_id}
 531        self.client.post(delete_url, params)
 532        count = ParsedInstance.query_mongo(
 533            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 534            count=True)[0]['count']
 535        self.assertEqual(count, initial_count + 1)
 536        # create the export
 537        csv_export_url = reverse(
 538            'csv_export', kwargs={"username": self.user.username,
 539                                "id_string":self.xform.id_string})
 540        response = self.client.get(csv_export_url)
 541        self.assertEqual(response.status_code, 200)
 542        f = StringIO.StringIO(self._get_response_content(response))
 543        csv_reader = csv.reader(f)
 544        num_rows = len([row for row in csv_reader])
 545        f.close()
 546        # number of rows == 2 i.e. initial_count + header plus one row
 547        self.assertEqual(num_rows, initial_count + 2)
 548
 549    def test_edited_submissions_in_exports(self):
 550        self._publish_transportation_form()
 551        initial_count = ParsedInstance.query_mongo(
 552            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 553            count=True)[0]['count']
 554        instance_name = 'transport_2011-07-25_19-05-36'
 555        path = os.path.join(
 556            'main', 'tests', 'fixtures', 'transportation', 'instances_w_uuid',
 557            instance_name, instance_name + '.xml')
 558        self._make_submission(path)
 559        count = ParsedInstance.query_mongo(
 560            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 561            count=True)[0]['count']
 562        self.assertEqual(count, initial_count+1)
 563        instance = Instance.objects.filter(
 564            xform=self.xform).order_by('id').reverse()[0]
 565        # make edited submission - simulating what enketo would return
 566        instance_name = 'transport_2011-07-25_19-05-36-edited'
 567        path = os.path.join(
 568            'main', 'tests', 'fixtures', 'transportation', 'instances_w_uuid',
 569            instance_name, instance_name + '.xml')
 570        self._make_submission(path)
 571        count = ParsedInstance.query_mongo(
 572            self.user.username, self.xform.id_string, '{}', '[]', '{}',
 573            count=True)[0]['count']
 574        self.assertEqual(count, initial_count+1)
 575        # create the export
 576        csv_export_url = reverse(
 577            'csv_export', kwargs={"username": self.user.username,
 578                                "id_string":self.xform.id_string})
 579        response = self.client.get(csv_export_url)
 580        self.assertEqual(response.status_code, 200)
 581        f = StringIO.StringIO(self._get_response_content(response))
 582        csv_reader = csv.DictReader(f)
 583        data = [row for row in csv_reader]
 584        f.close()
 585        num_rows = len(data)
 586        # number of rows == initial_count + 1
 587        self.assertEqual(num_rows, initial_count + 1)
 588        key ='transport/loop_over_transport_types_frequency/ambulance/frequency_to_referral_facility'
 589        self.assertEqual(data[initial_count][key], "monthly")
 590
 591    def test_export_ids_dont_have_comma_separation(self):
 592        """
 593        It seems using {{ }} to output numbers greater than 1000 formats the
 594        number with a thousand separator
 595        """
 596        self._publish_transportation_form()
 597        self._submit_transport_instance()
 598        # create an in-complete export
 599        export = Export.objects.create(id=1234, xform=self.xform,
 600                              export_type=Export.XLS_EXPORT)
 601        self.assertEqual(export.pk, 1234)
 602        export_list_url = reverse(
 603            export_list, kwargs={
 604                "username": self.user.username,
 605                "id_string": self.xform.id_string,
 606                "export_type": Export.XLS_EXPORT
 607            })
 608        response = self.client.get(export_list_url)
 609        self.assertContains(response, '#delete-1234"')
 610        self.assertNotContains(response, '#delete-1,234"')
 611
 612    def test_export_progress_updates(self):
 613        """
 614        Test that after generate_export is called, we change out state to
 615        pending and after its complete, we change it to complete, if we fail
 616        between the two, updates, we have failed
 617        """
 618        self._publish_transportation_form()
 619        # generate an export that fails because of the NoRecordsFound exception
 620        export = Export.objects.create(xform=self.xform,
 621            export_type=Export.XLS_EXPORT)
 622        # check that progress url says pending
 623        progress_url = reverse(export_progress, kwargs={
 624            'username': self.user.username,
 625            'id_string': self.xform.id_string,
 626            'export_type': 'xls'
 627        })
 628        params = {'export_ids': [export.id]}
 629        response = self.client.get(progress_url, params)
 630        status = json.loads(response.content)[0]
 631        self.assertEqual(status["complete"], False)
 632        self.assertEqual(status["filename"], None)
 633
 634        export.internal_status = Export.FAILED
 635        export.save()
 636        # check that progress url says failed
 637        progress_url = reverse(export_progress, kwargs={
 638            'username': self.user.username,
 639            'id_string': self.xform.id_string,
 640            'export_type': 'xls'
 641        })
 642        params = {'export_ids': [export.id]}
 643        response = self.client.get(progress_url, params)
 644        status = json.loads(response.content)[0]
 645        self.assertEqual(status["complete"], True)
 646        self.assertEqual(status["filename"], None)
 647
 648        # make a submission and create a valid export
 649        self._submit_transport_instance()
 650        create_xls_export(
 651            self.user.username,
 652            self.xform.id_string, export.id)
 653        params = {'export_ids': [export.id]}
 654        response = self.client.get(progress_url, params)
 655        status = json.loads(response.content)[0]
 656        self.assertEqual(status["complete"], True)
 657        self.assertIsNotNone(status["filename"])
 658
 659    def test_direct_export_returns_newset_export_if_not_updated_since(self):
 660        self._publish_transportation_form()
 661        self._submit_transport_instance()
 662        self.assertEqual(self.response.status_code, 201)
 663        sleep(1)
 664        self._submit_transport_instance_w_uuid("transport_2011-07-25_19-05-36")
 665        self.assertEqual(self.response.status_code, 201)
 666
 667        initial_num_csv_exports = Export.objects.filter(
 668            xform=self.xform, export_type=Export.CSV_EXPORT).count()
 669        initial_num_xls_exports = Export.objects.filter(
 670            xform=self.xform, export_type=Export.XLS_EXPORT).count()
 671        # request a direct csv export
 672        csv_export_url = reverse('csv_export', kwargs={
 673            'username': self.user.username,
 674            'id_string': self.xform.id_string
 675        })
 676        xls_export_url = reverse('xls_export', kwargs={
 677            'username': self.user.username,
 678            'id_string': self.xform.id_string
 679        })
 680        response = self.client.get(csv_export_url)
 681        self.assertEqual(response.status_code, 200)
 682        # we should have initial_num_exports + 1 exports
 683        num_csv_exports = Export.objects.filter(
 684            xform=self.xform, export_type=Export.CSV_EXPORT).count()
 685        self.assertEqual(num_csv_exports, initial_num_csv_exports + 1)
 686
 687        # request another export without changing the data
 688        response = self.client.get(csv_export_url)
 689        self.assertEqual(response.status_code, 200)
 690        # we should still only have a single export object
 691        num_csv_exports = Export.objects.filter(
 692            xform=self.xform, export_type=Export.CSV_EXPORT).count()
 693        self.assertEqual(num_csv_exports, initial_num_csv_exports + 1)
 694
 695        # this should not affect a direct XLS export and XLS should still re-generate
 696        response = self.client.get(xls_export_url)
 697        self.assertEqual(response.status_code, 200)
 698        num_xls_exports = Export.objects.filter(
 699            xform=self.xform, export_type=Export.XLS_EXPORT).count()
 700        self.assertEqual(num_xls_exports, initial_num_xls_exports + 1)
 701
 702        # make sure xls doesnt re-generate if data hasn't changed
 703        response = self.client.get(xls_export_url)
 704        self.assertEqual(response.status_code, 200)
 705        num_xls_exports = Export.objects.filter(
 706            xform=self.xform, export_type=Export.XLS_EXPORT).count()
 707        self.assertEqual(num_xls_exports, initial_num_xls_exports + 1)
 708
 709        sleep(1)
 710        # check that data edits cause a re-generation
 711        self._submit_transport_instance_w_uuid(
 712            "transport_2011-07-25_19-05-36-edited")
 713        self.assertEqual(self.response.status_code, 201)
 714        self.client.get(csv_export_url)
 715        self.assertEqual(response.status_code, 200)
 716        # we should have an extra export now that the data has been updated
 717        num_csv_exports = Export.objects.filter(
 718            xform=self.xform, export_type=Export.CSV_EXPORT).count()
 719        self.assertEqual(num_csv_exports, initial_num_csv_exports + 2)
 720
 721        sleep(1)
 722        # and when we delete
 723        delete_url = reverse(delete_data, kwargs={
 724            'username': self.user.username,
 725            'id_string': self.xform.id_string
 726        })
 727        instance = Instance.objects.filter().order_by('-pk')[0]
 728        response = self.client.post(delete_url, {'id': instance.id})
 729        self.assertEqual(response.status_code, 200)
 730        response = self.client.get(csv_export_url)
 731        self.assertEqual(response.status_code, 200)
 732        # we should have an extra export now that the data has been updated by the delete
 733        num_csv_exports = Export.objects.filter(
 734            xform=self.xform, export_type=Export.CSV_EXPORT).count()
 735        self.assertEqual(num_csv_exports, initial_num_csv_exports + 3)
 736
 737    def test_exports_outdated_doesnt_consider_failed_exports(self):
 738        self._publish_transportation_form()
 739        self._submit_transport_instance()
 740        # create a bad export
 741        export = Export.objects.create(
 742            xform=self.xform, export_type=Export.XLS_EXPORT,
 743            internal_status=Export.FAILED)
 744        self.assertTrue(
 745            Export.exports_outdated(self.xform, export.export_type))
 746
 747    def test_exports_outdated_considers_pending_exports(self):
 748        self._publish_transportation_form()
 749        self._submit_transport_instance()
 750        # create a pending export
 751        export = Export.objects.create(
 752            xform=self.xform, export_type=Export.XLS_EXPORT,
 753            internal_status=Export.PENDING)
 754        self.assertFalse(
 755            Export.exports_outdated(self.xform, export.export_type))
 756
 757    def _get_csv_data(self, filepath):
 758        storage = get_storage_class()()
 759        csv_file = storage.open(filepath)
 760        reader = csv.DictReader(csv_file)
 761        data = reader.next()
 762        csv_file.close()
 763        return data
 764
 765    def _get_xls_data(self, filepath):
 766        storage = get_storage_class()()
 767        with storage.open(filepath) as f:
 768            workbook = open_workbook(file_contents=f.read())
 769        transportation_sheet = workbook.sheet_by_name("transportation")
 770        self.assertTrue(transportation_sheet.nrows > 1)
 771        headers = transportation_sheet.row_values(0)
 772        column1 = transportation_sheet.row_values(1)
 773        return dict(zip(headers, column1))
 774
 775    def test_column_header_delimiter_export_option(self):
 776        self._publish_transportation_form()
 777        # survey 1 has ambulance and bicycle as values for
 778        # transport/available_transportation_types_to_referral_facility
 779        self._submit_transport_instance(survey_at=1)
 780        create_csv_export_url = reverse(create_export, kwargs={
 781            'username': self.user.username,
 782            'id_string': self.xform.id_string,
 783            'export_type': 'csv'
 784        })
 785        default_params = {}
 786        custom_params = {
 787            'options[group_delimiter]': '.',
 788        }
 789        # test csv with default group delimiter
 790        response = self.client.post(create_csv_export_url, default_params)
 791        self.assertEqual(response.status_code, 302)
 792        export = Export.objects.filter(
 793            xform=self.xform, export_type='csv').latest('created_on')
 794        self.assertTrue(bool(export.filepath))
 795        data = self._get_csv_data(export.filepath)
 796        self.assertTrue(
 797            data.has_key(
 798                'transport/available_transportation_types_to_referral_facility/ambulance'))
 799        self.assertEqual(
 800            data['transport/available_transportation_types_to_referral_facility/ambulance'], 'True')
 801
 802        sleep(1)
 803        # test csv with dot delimiter
 804        response = self.client.post(create_csv_export_url, custom_params)
 805        self.assertEqual(response.status_code, 302)
 806        export = Export.objects.filter(
 807            xform=self.xform, export_type='csv').latest('created_on')
 808        self.assertTrue(bool(export.filepath))
 809        data = self._get_csv_data(export.filepath)
 810        self.assertTrue(
 811            data.has_key(
 812                'transport.available_transportation_types_to_referral_facility.ambulance'))
 813        self.assertEqual(
 814            data['transport.available_transportation_types_to_referral_facility.ambulance'], 'True')
 815
 816        # test xls with default group delimiter
 817        create_csv_export_url = reverse(create_export, kwargs={
 818            'username': self.user.username,
 819            'id_string': self.xform.id_string,
 820            'export_type': 'xls'
 821        })
 822        response = self.client.post(create_csv_export_url, default_params)
 823        self.assertEqual(response.status_code, 302)
 824        export = Export.objects.filter(
 825            xform=self.xform, export_type='xls').latest('created_on')
 826        self.assertTrue(bool(export.filepath))
 827        data = self._get_xls_data(export.filepath)
 828        self.assertTrue(
 829            data.has_key("transport/available_transportation_types_to_referral_facility/ambulance"))
 830        # xlrd reader seems to convert bools into integers i.e. 0 or 1
 831        self.assertEqual(
 832            data["transport/available_transportation_types_to_referral_facility/ambulance"], 1)
 833
 834        sleep(1)
 835        # test xls with dot delimiter
 836        response = self.client.post(create_csv_export_url, custom_params)
 837        self.assertEqual(response.status_code, 302)
 838        export = Export.objects.filter(
 839            xform=self.xform, export_type='xls').latest('created_on')
 840        self.assertTrue(bool(export.filepath))
 841        data = self._get_xls_data(export.filepath)
 842        self.assertTrue(
 843            data.has_key("transport.available_transportation_types_to_referral_facility.ambulance"))
 844        # xlrd reader seems to convert bools into integers i.e. 0 or 1
 845        self.assertEqual(
 846            data["transport.available_transportation_types_to_referral_facility.ambulance"], 1)
 847
 848    def test_split_select_multiple_export_option(self):
 849        self._publish_transportation_form()
 850        self._submit_transport_instance(survey_at=1)
 851        create_csv_export_url = reverse(create_export, kwargs={
 852            'username': self.user.username,
 853            'id_string': self.xform.id_string,
 854            'export_type': 'csv'
 855        })
 856        default_params = {}
 857        custom_params = {
 858            'options[dont_split_select_multiples]': 'yes'
 859        }
 860        # test csv with default split select multiples
 861        response = self.client.post(create_csv_export_url, default_params)
 862        self.assertEqual(response.status_code, 302)
 863        export = Export.objects.filter(
 864            xform=self.xform, export_type='csv').latest('created_on')
 865        self.assertTrue(bool(export.filepath))
 866        data = self._get_csv_data(export.filepath)
 867        # we should have transport/available_transportation_types_to_referral_facility/ambulance as a separate column
 868        self.assertTrue(
 869            data.has_key(
 870                'transport/available_transportation_types_to_referral_facility/ambulance'))
 871
 872        sleep(1)
 873        # test csv without default split select multiples
 874        response = self.client.post(create_csv_export_url, custom_params)
 875        self.assertEqual(response.status_code, 302)
 876        export = Export.objects.filter(
 877            xform=self.xform, export_type='csv').latest('created_on')
 878        self.assertTrue(bool(export.filepath))
 879        data = self._get_csv_data(export.filepath)
 880        # transport/available_transportation_types_to_referral_facility/ambulance should not be in its own column
 881        self.assertFalse(
 882            data.has_key(
 883                'transport/available_transportation_types_to_referral_facility/ambulance'))
 884        # transport/available_transportation_types_to_referral_facility should be a column
 885        self.assertTrue(
 886            data.has_key(
 887                'transport/available_transportation_types_to_referral_facility'))
 888        # check that ambulance is one the values within the transport/available_transportation_types_to_referral_facility column
 889        self.assertTrue("ambulance" in data['transport/available_transportation_types_to_referral_facility'].split(" "))
 890
 891        create_xls_export_url = reverse(create_export, kwargs={
 892            'username': self.user.username,
 893            'id_string': self.xform.id_string,
 894            'export_type': 'xls'
 895        })
 896        # test xls with default split select multiples
 897        response = self.client.post(create_xls_export_url, default_params)
 898        self.assertEqual(response.status_code, 302)
 899        export = Export.objects.filter(
 900            xform=self.xform, export_type='xls').latest('created_on')
 901        self.assertTrue(bool(export.filepath))
 902        data = self._get_xls_data(export.filepath)
 903        # we should have transport/available_transportation_types_to_referral_facility/ambulance as a separate column
 904        self.assertTrue(
 905            data.has_key(
 906                'transport/available_transportation_types_to_referral_facility/ambulance'))
 907
 908        sleep(1)
 909        # test xls without default split select multiples
 910        response = self.client.post(create_xls_export_url, custom_params)
 911        self.assertEqual(response.status_code, 302)
 912        export = Export.objects.filter(
 913            xform=self.xform, export_type='xls').latest('created_on')
 914        self.assertTrue(bool(export.filepath))
 915        data = self._get_xls_data(export.filepath)
 916        # transport/available_transportation_types_to_referral_facility/ambulance should NOT be in its own column
 917        self.assertFalse(
 918            data.has_key(
 919                'transport/available_transportation_types_to_referral_facility/ambulance'))
 920        # transport/available_transportation_types_to_referral_facility should be a column
 921        self.assertTrue(
 922            data.has_key(
 923                'transport/available_transportation_types_to_referral_facility'))
 924        # check that ambulance is one the values within the transport/available_transportation_types_to_referral_facility column
 925        self.assertTrue("ambulance" in data['transport/available_transportation_types_to_referral_facility'].split(" "))
 926
 927    def test_dict_to_joined_export_works(self):
 928        data =\
 929            {
 930                'name': 'Abe',
 931                'age': '35',
 932                '_geolocation': [None, None],
 933                'attachments': ['abcd.jpg', 'efgh.jpg'],
 934                'children':
 935                [
 936                    {
 937                        'children/name': 'Mike',
 938                        'children/age': '5',
 939                        'children/cartoons':
 940                        [
 941                            {
 942                                'children/cartoons/name': 'Tom & Jerry',
 943                                'children/cartoons/why': 'Tom is silly',
 944                            },
 945                            {
 946                                'children/cartoons/name': 'Flinstones',
 947                                'children/cartoons/why': u"I like bamb bam\u0107",
 948                            }
 949                        ]
 950                    },
 951                    {
 952                        'children/name': 'John',
 953                        'children/age': '2',
 954                        'children/cartoons':[]
 955                    },
 956                    {
 957                        'children/name': 'Imora',
 958                        'children/age': '3',
 959                        'children/cartoons':
 960                        [
 961                            {
 962                                'children/cartoons/name': 'Shrek',
 963                                'children/cartoons/why': 'He\'s so funny'
 964                            },
 965                            {
 966                                'children/cartoons/name': 'Dexter\'s Lab',
 967                                'children/cartoons/why': 'He thinks hes smart',
 968                                'children/cartoons/characters':
 969                                [
 970                                    {
 971                                        'children/cartoons/characters/name': 'Dee Dee',
 972                                        'children/cartoons/characters/good_or_evil': 'good'
 973                                    },
 974                                    {
 975                                        'children/cartoons/characters/name': 'Dexter',
 976                                        'children/cartoons/characters/good_or_evil': 'evil'
 977                                    },
 978                                ]
 979                            }
 980                        ]
 981                    }
 982                ]
 983            }
 984        expected_output =\
 985            {
 986                'survey': {
 987                  'name': 'Abe',
 988                  'age': '35'
 989                },
 990                'children':
 991                [
 992                    {
 993                        'children/name': 'Mike',
 994                        'children/age': '5',
 995                        '_index': 1,
 996                        '_parent_table_name': 'survey',
 997                        '_parent_index': 1
 998                    },
 999                    {
1000                        'children/name': 'John',
1001                        'children/age': '2',
1002                        '_index': 2,
1003                        '_parent_table_name': 'survey',
1004                        '_parent_index': 1
1005                    },
1006                    {
1007                        'children/name': 'Imora',
1008                        'children/age': '3',
1009                        '_index': 3,
1010                        '_parent_table_name': 'survey',
1011                        '_parent_index': 1
1012                    },
1013                ],
1014                'children/cartoons':
1015                [
1016                    {
1017                        'children/cartoons/name': 'Tom & Jerry',
1018                        'children/cartoons/why': 'Tom is silly',
1019                        '_index': 1,
1020                        '_parent_table_name': 'children',
1021                        '_parent_index': 1
1022                    },
1023                    {
1024                        'children/cartoons/name': 'Flinstones',
1025                        'children/cartoons/why': u"I like bamb bam\u0107",
1026                        '_index': 2,
1027                        '_parent_table_name': 'children',
1028                        '_parent_index': 1
1029                    },
1030                    {
1031                        'children/cartoons/name': 'Shrek',
1032                        'children/cartoons/why': 'He\'s so funny',
1033                        '_index': 3,
1034                        '_parent_table_name': 'children',
1035                        '_parent_index': 3
1036                    },
1037                    {
1038                        'children/cartoons/name': 'Dexter\'s Lab',
1039                        'children/cartoons/why': 'He thinks hes smart',
1040                        '_index': 4,
1041                        '_parent_table_name': 'children',
1042                        '_parent_index': 3
1043                    }
1044                ],
1045                'children/cartoons/characters':
1046                [
1047                    {
1048                        'children/cartoons/characters/name': 'Dee Dee',
1049                        'children/cartoons/characters/good_or_evil': 'good',
1050                        '_index': 1,
1051                        '_parent_table_name': 'children/cartoons',
1052                        '_parent_index': 4
1053                    },
1054                    {
1055                        'children/cartoons/characters/name': 'Dexter',
1056                        'children/cartoons/characters/good_or_evil': 'evil',
1057                        '_index': 2,
1058                        '_parent_table_name': 'children/cartoons',
1059                        '_parent_index': 4
1060                    }
1061                ]
1062            }
1063        survey_name = 'survey'
1064        indices = {survey_name: 0}
1065        output = dict_to_joined_export(data, 1, indices, survey_name)
1066        self.assertEqual(output[survey_name], expected_output[survey_name])
1067        # 1st level
1068        self.assertEqual(len(output['children']), 3)
1069        for child in enumerate(['Mike', 'John', 'Imora']):
1070            index = child[0]
1071            name = child[1]
1072            self.assertEqual(
1073                filter(
1074                    lambda x: x['children/name'] == name,
1075                    output['children'])[0],
1076                expected_output['children'][index])
1077        # 2nd level
1078        self.assertEqual(len(output['children/cartoons']), 4)
1079        for cartoon in enumerate(
1080                ['Tom & Jerry', 'Flinstones', 'Shrek', 'Dexter\'s Lab']):
1081            index = cartoon[0]
1082            name = cartoon[1]
1083            self.assertEqual(
1084                filter(
1085                    lambda x: x['children/cartoons/name'] == name,
1086                    output['children/cartoons'])[0],
1087                expected_output['children/cartoons'][index])
1088        # 3rd level
1089        self.assertEqual(len(output['children/cartoons/characters']), 2)
1090        for characters in enumerate(['Dee Dee', 'Dexter']):
1091            index = characters[0]
1092            name = characters[1]
1093            self.assertEqual(
1094                filter(
1095                    lambda x: x['children/cartoons/characters/name'] == name,
1096                    output['children/cartoons/characters'])[0],
1097                expected_output['children/cartoons/characters'][index])
1098
1099    def test_generate_csv_zip_export(self):
1100        # publish xls form
1101        self._publish_transportation_form_and_submit_instance()
1102        # create export db object
1103        export = generate_export(
1104            Export.CSV_ZIP_EXPORT, "zip", self.user.username,
1105            self.xform.id_string, group_delimiter='/',
1106            split_select_multiples=True)
1107        storage = get_storage_class()()
1108        self.assertTrue(storage.exists(export.filepath))
1109        path, ext = os.path.splitext(export.filename)
1110        self.assertEqual(ext, '.zip')
1111
1112
1113class TestExportBuilder(MainTestCase):
1114    data = [
1115        {
1116            'name': 'Abe',
1117            'age': 35,
1118            'tel/telLg==office': '020123456',
1119            'children':
1120            [
1121                {
1122                    'children/name': 'Mike',
1123                    'children/age': 5,
1124                    'children/fav_colors': 'red blue',
1125                    'children/iceLg==creams': 'vanilla chocolate',
1126                    'children/cartoons':
1127                    [
1128                        {
1129                            'children/cartoons/name': 'Tom & Jerry',
1130               

Large files files are truncated, but you can click here to view the full file