/corehq/apps/userreports/reports/builder/forms.py
Python | 1226 lines | 1100 code | 70 blank | 56 comment | 58 complexity | 388565bec0bfd96a4bba848703c2a466 MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-2.1
- import datetime
- import uuid
- from abc import ABCMeta, abstractmethod
- from collections import OrderedDict, namedtuple
- from django import forms
- from django.conf import settings
- from django.utils.translation import gettext as _
- from crispy_forms import layout as crispy
- from crispy_forms.bootstrap import StrictButton
- from crispy_forms.helper import FormHelper
- from memoized import memoized
- from corehq.apps.app_manager.app_schemas.case_properties import (
- get_case_properties,
- )
- from corehq.apps.app_manager.fields import ApplicationDataSourceUIHelper
- from corehq.apps.app_manager.models import Application
- from corehq.apps.app_manager.xform import XForm
- from corehq.apps.case_search.const import COMMCARE_PROJECT
- from corehq.apps.data_dictionary.util import get_data_dict_props_by_case_type
- from corehq.apps.domain.models import DomainAuditRecordEntry
- from corehq.apps.hqwebapp import crispy as hqcrispy
- from corehq.apps.registry.helper import DataRegistryHelper
- from corehq.apps.registry.utils import RegistryPermissionCheck
- from corehq.apps.userreports import tasks
- from corehq.apps.userreports.app_manager.data_source_meta import (
- APP_DATA_SOURCE_TYPE_VALUES,
- DATA_SOURCE_TYPE_RAW,
- REPORT_BUILDER_DATA_SOURCE_TYPE_VALUES,
- DATA_SOURCE_TYPE_CASE, DATA_SOURCE_TYPE_FORM,
- make_case_data_source_filter, make_form_data_source_filter,
- )
- from corehq.apps.userreports.app_manager.helpers import clean_table_name
- from corehq.apps.userreports.const import DATA_SOURCE_MISSING_APP_ERROR_MESSAGE, LENIENT_MAXIMUM_EXPANSION
- from corehq.apps.userreports.exceptions import BadBuilderConfigError
- from corehq.apps.userreports.models import (
- DataSourceBuildInformation,
- DataSourceConfiguration,
- DataSourceMeta,
- ReportConfiguration,
- ReportMeta,
- get_datasource_config_infer_type,
- guess_data_source_type,
- RegistryDataSourceConfiguration, RegistryReportConfiguration,
- )
- from corehq.apps.userreports.reports.builder import (
- DEFAULT_CASE_PROPERTY_DATATYPES,
- FORM_METADATA_PROPERTIES,
- get_filter_format_from_question_type,
- const)
- from corehq.apps.userreports.reports.builder.columns import (
- CasePropertyColumnOption,
- CountColumn,
- FormMetaColumnOption,
- MultiselectQuestionColumnOption,
- OwnernameComputedCasePropertyOption,
- QuestionColumnOption,
- RawPropertyColumnOption,
- UsernameComputedCasePropertyOption,
- )
- from corehq.apps.userreports.reports.builder.const import (
- COMPUTED_OWNER_LOCATION_PROPERTY_ID,
- COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID,
- COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID,
- COMPUTED_OWNER_NAME_PROPERTY_ID,
- COMPUTED_USER_NAME_PROPERTY_ID,
- PROPERTY_TYPE_CASE_PROP,
- PROPERTY_TYPE_META,
- PROPERTY_TYPE_QUESTION,
- PROPERTY_TYPE_RAW,
- UCR_AGG_AVG,
- UCR_AGG_EXPAND,
- UCR_AGG_SIMPLE,
- UCR_AGG_SUM,
- UI_AGG_AVERAGE,
- UI_AGG_COUNT_PER_CHOICE,
- UI_AGG_GROUP_BY,
- UI_AGG_SUM,
- )
- from corehq.apps.userreports.reports.builder.filter_formats import get_pre_filter_format
- from corehq.apps.userreports.reports.builder.sources import (
- get_source_type_from_report_config,
- )
- from corehq.apps.userreports.sql import get_column_name
- from corehq.apps.userreports.ui.fields import JsonField
- from corehq.apps.userreports.util import has_report_builder_access, get_ucr_datasource_config_by_id
- from corehq.toggles import (
- SHOW_RAW_DATA_SOURCES_IN_REPORT_BUILDER,
- SHOW_OWNER_LOCATION_PROPERTY_IN_REPORT_BUILDER,
- OVERRIDE_EXPANDED_COLUMN_LIMIT_IN_REPORT_BUILDER,
- SHOW_IDS_IN_REPORT_BUILDER,
- DATA_REGISTRY_UCR
- )
- from dimagi.utils.couch.undo import undo_delete
- STATIC_CASE_PROPS = [
- "closed",
- "closed_on",
- "modified_on",
- "name",
- "opened_on",
- "owner_id",
- "user_id",
- ]
- # PostgreSQL limit = 1600. Sane limit = 500?
- MAX_COLUMNS = 500
- TEMP_DATA_SOURCE_LIFESPAN = 24 * 60 * 60
- SAMPLE_DATA_MAX_ROWS = 100
- class FilterField(JsonField):
- """
- A form field with a little bit of validation for report builder report
- filter configuration.
- """
- def validate(self, value):
- super(FilterField, self).validate(value)
- for filter_conf in value:
- if filter_conf.get('format', None) not in (list(const.REPORT_BUILDER_FILTER_TYPE_MAP) + [""]):
- raise forms.ValidationError("Invalid filter format!")
- class DataSourceProperty(object):
- """
- A container class for information about data source properties
- Class attributes:
- type -- either "case_property", "question", "meta", or "raw"
- id -- A string that uniquely identifies this property. For question based
- properties this is the question id, for case based properties this is
- the case property name.
- text -- A human readable representation of the property source. For
- questions this is the question label.
- source -- For questions, this is a dict representing the question as returned
- by Xform.get_questions(), for case properties and form metadata it is just
- the name of the property.
- data_types
- """
- def __init__(self, type, id, text, source, data_types):
- self._type = type
- self._id = id
- self._text = text
- self._source = source
- self._data_types = data_types
- def to_view_model(self):
- """
- Return a dictionary representation to be used by the js
- """
- return {
- "type": self._type,
- "id": self._id,
- "text": self._text,
- "source": self._source,
- }
- def get_text(self):
- return self._text
- def get_type(self):
- return self._type
- def get_id(self):
- return self._id
- def get_source(self):
- return self._source
- def to_report_column_option(self):
- if self._type == PROPERTY_TYPE_QUESTION:
- if self._source['type'] == "MSelect":
- return MultiselectQuestionColumnOption(self._id, self._text, self._source)
- else:
- return QuestionColumnOption(self._id, self._data_types, self._text, self._source)
- elif self._type == PROPERTY_TYPE_META:
- return FormMetaColumnOption(self._id, self._data_types, self._text, self._source)
- elif self._type == PROPERTY_TYPE_CASE_PROP:
- if self._id in (
- COMPUTED_OWNER_NAME_PROPERTY_ID,
- COMPUTED_OWNER_LOCATION_PROPERTY_ID,
- COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID,
- COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID
- ):
- return OwnernameComputedCasePropertyOption(self._id, self._data_types, self._text)
- elif self._id == COMPUTED_USER_NAME_PROPERTY_ID:
- return UsernameComputedCasePropertyOption(self._id, self._data_types, self._text)
- else:
- return CasePropertyColumnOption(self._id, self._data_types, self._text)
- else:
- assert self._type == PROPERTY_TYPE_RAW
- return RawPropertyColumnOption(self._id, self._data_types, self._text, self._source)
- def _get_filter_format(self, filter_configuration):
- """
- Return the UCR filter type that should be used for the given filter configuration (passed from the UI).
- """
- selected_filter_type = filter_configuration['format']
- if not selected_filter_type or self._type in ('question', 'meta'):
- if self._type == 'question':
- filter_format = get_filter_format_from_question_type(self._source['type'])
- else:
- assert self._type == 'meta'
- filter_format = get_filter_format_from_question_type(self._source[1])
- else:
- filter_format = const.REPORT_BUILDER_FILTER_TYPE_MAP[selected_filter_type]
- return filter_format
- def _get_ui_aggregation_for_filter_format(self, filter_format):
- """
- ColumnOption._get_indicator(aggregation) uses the aggregation type to determine what data type the
- indicator should be. Therefore, we need to convert filter formats to aggregation types so that we can
- create the correct type of indicator.
- """
- if filter_format == "numeric":
- return UI_AGG_SUM # This could also be UI_AGG_AVERAGE, just needs to force numeric
- else:
- return UI_AGG_GROUP_BY
- def to_report_filter(self, configuration, index):
- """
- Return a UCR report filter configuration for the given configuration.
- :param configuration: dictionary representing options selected in UI.
- :param index: Index of this filter in the list of filters configured by the user.
- :return:
- """
- filter_format = self._get_filter_format(configuration)
- ui_aggregation = self._get_ui_aggregation_for_filter_format(filter_format)
- column_id = self.to_report_column_option().get_indicators(ui_aggregation)[0]['column_id']
- filter = {
- "field": column_id,
- "slug": "{}_{}".format(column_id, index),
- "display": configuration["display_text"],
- "type": filter_format
- }
- if configuration['format'] == const.FORMAT_DATE:
- filter.update({'compare_as_string': True})
- if filter_format == 'dynamic_choice_list' and self._id == COMPUTED_OWNER_NAME_PROPERTY_ID:
- filter.update({"choice_provider": {"type": "owner"}})
- if filter_format == 'dynamic_choice_list' and self._id == COMPUTED_USER_NAME_PROPERTY_ID:
- filter.update({"choice_provider": {"type": "user"}})
- if filter_format == 'dynamic_choice_list' and self._id == COMPUTED_OWNER_LOCATION_PROPERTY_ID:
- filter.update({"choice_provider": {"type": "location"}})
- if filter_format == 'dynamic_choice_list' and self._id == COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID:
- filter.update({"choice_provider": {"type": "location", "include_descendants": True}})
- if filter_format == 'dynamic_choice_list' and self._id == COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID:
- filter.update({"choice_provider": {"type": "location", "include_descendants": True, "show_all_locations": True}})
- if filter_format == 'dynamic_choice_list' and self._id == COMMCARE_PROJECT:
- filter.update({"choice_provider": {"type": COMMCARE_PROJECT}})
- if configuration.get('pre_value') or configuration.get('pre_operator'):
- filter.update({
- 'type': 'pre', # type could have been "date"
- 'pre_operator': configuration.get('pre_operator', None),
- 'pre_value': configuration.get('pre_value', []),
- })
- if configuration['format'] == const.PRE_FILTER_VALUE_IS_EMPTY:
- filter.update({
- 'type': 'pre',
- 'pre_operator': "",
- 'pre_value': "", # for now assume strings - this may not always work but None crashes
- })
- if configuration['format'] == const.PRE_FILTER_VALUE_EXISTS:
- filter.update({
- 'type': 'pre',
- 'pre_operator': "!=",
- 'pre_value': "",
- })
- if configuration['format'] == const.PRE_FILTER_VALUE_NOT_EQUAL:
- filter.update({
- 'type': 'pre',
- 'pre_operator': "distinct from",
- # pre_value already set by "pre" clause
- })
- return filter
- def to_report_filter_indicator(self, configuration):
- """
- Return the indicator that would correspond to the given filter configuration
- """
- filter_format = self._get_filter_format(configuration)
- ui_aggregation = self._get_ui_aggregation_for_filter_format(filter_format)
- return self.to_report_column_option()._get_indicator(ui_aggregation)
- class ReportBuilderDataSourceInterface(metaclass=ABCMeta):
- """
- Abstract interface to a data source in report builder.
- A data source could be an (app, form), (app, case_type), or (registry, case_type) pair (see
- ManagedReportBuilderDataSourceHelper), or it can be a real UCR data source (see UnmanagedDataSourceHelper)
- """
- @property
- @abstractmethod
- def report_config_class(self):
- """Return the report class type"""
- raise NotImplementedError
- @property
- @abstractmethod
- def uses_managed_data_source(self):
- """
- Whether this interface uses a managed data source.
- If true, the data source will be created / modified with the report, and the
- temporary data source workflow will be enabled.
- If false, the data source is assumed to exist and be available as self.data_source_id.
- :return:
- """
- raise NotImplementedError
- @property
- @abstractmethod
- def data_source_properties(self):
- """
- A dictionary containing the various properties that may be used as indicators
- or columns in the data source or report.
- Keys are strings that uniquely identify properties.
- Values are DataSourceProperty instances.
- >> self.data_source_properties
- {
- "/data/question1": DataSourceProperty(
- type="question",
- id="/data/question1",
- text="Enter the child's name",
- source={
- 'repeat': None,
- 'group': None,
- 'value': '/data/question1',
- 'label': 'question1',
- 'tag': 'input',
- 'type': 'Text'
- },
- data_types=["string"]
- ),
- "meta/deviceID": DataSourceProperty(
- type="meta",
- id="meta/deviceID",
- text="deviceID",
- source=("deviceID", "string"),
- data_types=["string"]
- )
- }
- """
- pass
- @property
- @abstractmethod
- def report_column_options(self):
- pass
- class ManagedReportBuilderDataSourceHelper(ReportBuilderDataSourceInterface):
- """Abstract class that represents the interface required for building managed
- data sources
- When configuring a report, one can use ManagedReportBuilderDataSourceHelper to determine some
- of the properties of the required report data source, such as:
- - referenced doc type
- - filter
- - indicators
- """
- def __init__(self, domain, source_type, source_id):
- assert (source_type in ['case', 'form'])
- self.domain = domain
- self.source_type = source_type
- # case type or form ID
- self.source_id = source_id
- @property
- def uses_managed_data_source(self):
- return True
- @property
- def report_config_class(self):
- return RegistryReportConfiguration if self.uses_registry_data_source else ReportConfiguration
- @property
- def uses_registry_data_source(self):
- """
- Whether this interface uses a registry data source.
- If true, it will use RegistryDataSourceConfiguration.
- If false, it uses DataSourceConfiguration.
- :return:
- """
- raise NotImplementedError
- @property
- @abstractmethod
- def source_doc_type(self):
- """Return the doc_type the datasource.referenced_doc_type"""
- raise NotImplementedError
- @property
- @abstractmethod
- def filter(self):
- """
- Return the filter configuration for the DataSourceConfiguration.
- """
- raise NotImplementedError
- @abstractmethod
- def base_item_expression(self, is_multiselect_chart_report, multiselect_field=None):
- raise NotImplementedError
- def indicators(self, columns, filters, as_dict=False):
- """
- Return a list of indicators to be used in a data source configuration that supports the given columns and
- indicators.
- :param columns: A list of objects representing columns in the report.
- Each object has a "property" and "calculation" key
- :param filters: A list of filter configuration objects
- """
- indicators = OrderedDict()
- for column in columns:
- # Property is only set if the column exists in report_column_options
- if column['property']:
- column_option = self.report_column_options[column['property']]
- for indicator in column_option.get_indicators(column['calculation']):
- # A column may have multiple indicators. e.g. "Group By" and "Count Per Choice" aggregations
- # will use one indicator for the field's string value, and "Sum" and "Average" aggregations
- # will use a second indicator for the field's numerical value. "column_id" includes the
- # indicator's data type, so it is unique per indicator ... except for choice list indicators,
- # because they get expanded to one column per choice. The column_id of choice columns will end
- # up unique because they will include a slug of the choice value. Here "column_id + type" is
- # unique.
- indicator_key = (indicator['column_id'], indicator['type'])
- indicators.setdefault(indicator_key, indicator)
- for filter_ in filters:
- # Property is only set if the filter exists in report_column_options
- if filter_['property']:
- property_ = self.data_source_properties[filter_['property']]
- indicator = property_.to_report_filter_indicator(filter_)
- indicator_key = (indicator['column_id'], indicator['type'])
- indicators.setdefault(indicator_key, indicator)
- if as_dict:
- return indicators
- return list(indicators.values())
- def all_possible_indicators(self, required_columns, required_filters):
- """
- Will generate a set of possible indicators for the datasource making sure to include the
- provided columns and filters
- """
- indicators = self.indicators(required_columns, required_filters, as_dict=True)
- for column_option in self.report_column_options.values():
- for agg in column_option.aggregation_options:
- for indicator in column_option.get_indicators(agg):
- indicator_key = (indicator['column_id'], indicator['type'])
- indicators.setdefault(indicator_key, indicator)
- return list(indicators.values())[:MAX_COLUMNS]
- @property
- @abstractmethod
- def data_source_properties(self):
- raise NotImplementedError
- @property
- @memoized
- def report_column_options(self):
- options = OrderedDict()
- for id_, prop in self.data_source_properties.items():
- options[id_] = prop.to_report_column_option()
- # NOTE: Count columns aren't useful for table reports. But we need it in the column options because
- # the options are currently static, after loading the report builder a user can switch to an aggregated
- # report.
- count_col = CountColumn("Number of Cases" if self.source_type == "case" else "Number of Forms")
- options[count_col.get_property()] = count_col
- return options
- @property
- @abstractmethod
- def data_source_name(self):
- raise NotImplementedError
- def construct_data_source(self, table_id, **kwargs):
- return DataSourceConfiguration(domain=self.domain, table_id=table_id, **kwargs)
- def _ds_config_kwargs(self, indicators, is_multiselect_chart_report=False, multiselect_field=None):
- if is_multiselect_chart_report:
- base_item_expression = self.base_item_expression(True, multiselect_field)
- else:
- base_item_expression = self.base_item_expression(False)
- return dict(
- display_name=self.data_source_name,
- referenced_doc_type=self.source_doc_type,
- configured_filter=self.filter,
- configured_indicators=indicators,
- base_item_expression=base_item_expression,
- meta=DataSourceMeta(build=self._get_data_source_build_information())
- )
- def _get_data_source_build_information(self):
- raise NotImplementedError
- def get_temp_datasource_constructor_kwargs(self, required_columns, required_filters):
- indicators = self._remove_defaults_from_indicators(
- self.all_possible_indicators(required_columns, required_filters)
- )
- return self._ds_config_kwargs(indicators)
- def get_datasource_constructor_kwargs(self, columns, filters,
- is_multiselect_chart_report=False, multiselect_field=None):
- indicators = self._remove_defaults_from_indicators(
- self.indicators(columns, filters)
- )
- return self._ds_config_kwargs(indicators, is_multiselect_chart_report, multiselect_field)
- def _remove_defaults_from_indicators(self, indicators):
- defaults = self._get_datasource_default_columns()
- return [
- indicator for indicator in indicators
- if indicator['column_id'] not in defaults
- ]
- def _get_datasource_default_columns(self):
- return {
- column.id
- for indicator in DataSourceConfiguration().default_indicators
- for column in indicator.get_columns()
- }
- class UnmanagedDataSourceHelper(ReportBuilderDataSourceInterface):
- """
- A ReportBuilderDataSourceInterface that encapsulates an existing data source.
- """
- def __init__(self, domain, app, source_type, source_id):
- assert source_type == 'data_source'
- self.domain = domain
- self.app = app
- self.source_type = source_type
- # source_id is the ID of a UCR data source
- self.data_source_id = source_id
- @property
- def uses_managed_data_source(self):
- return False
- @property
- def report_config_class(self):
- return {
- "DataSourceConfiguration": ReportConfiguration,
- "RegistryDataSourceConfiguration": RegistryReportConfiguration,
- }[self.data_source.doc_type]
- @property
- @memoized
- def data_source(self):
- return get_datasource_config_infer_type(self.data_source_id, self.domain)[0]
- @property
- def data_source_properties(self):
- def _data_source_property_from_ucr_column(column):
- # note: using column ID as the display text is a bummer but we don't have a a better
- # way to easily access a readable name for these yet
- return DataSourceProperty(
- type=PROPERTY_TYPE_RAW,
- id=column.id,
- text=column.id,
- source=column.id,
- data_types=[column.datatype],
- )
- properties = OrderedDict()
- for column in self.data_source.get_columns():
- properties[column.id] = _data_source_property_from_ucr_column(column)
- return properties
- @property
- def report_column_options(self):
- options = OrderedDict()
- for id_, prop in self.data_source_properties.items():
- options[id_] = prop.to_report_column_option()
- return options
- class ApplicationFormDataSourceHelper(ManagedReportBuilderDataSourceHelper):
- def __init__(self, domain, app, source_type, source_id):
- assert source_type == 'form'
- self.app = app
- super().__init__(domain, source_type, source_id)
- self.source_form = self.app.get_form(source_id)
- self.source_xform = XForm(self.source_form.source, domain=app.domain)
- def base_item_expression(self, is_multiselect_chart_report, multiselect_field=None):
- """
- Return the base_item_expression for the DataSourceConfiguration.
- Normally this is {}, but if this is a data source for a chart report that is aggregated by a multiselect
- question, then we want one row per multiselect answer.
- :param is_multiselect_chart_report: True if the data source will be used for a chart report aggregated by
- a multiselect question.
- :param multiselect_field: The field that the multiselect aggregated report is aggregated by.
- :return: A base item expression.
- """
- if not is_multiselect_chart_report:
- return {}
- else:
- assert multiselect_field, "multiselect_field is required if is_multiselect_chart_report is True"
- property = self.data_source_properties[multiselect_field]
- path = ['form'] + property.get_source()['value'].split('/')[2:]
- choices = [c['value'] for c in property.get_source()['options']]
- def sub_doc(path):
- if not path:
- return {"type": "property_name", "property_name": "choice"}
- else:
- return {
- "type": "dict",
- "properties": {
- path[0]: sub_doc(path[1:])
- }
- }
- return {
- "type": "map_items",
- "items_expression": {
- "type": "iterator",
- "expressions": [
- {
- "type": "dict",
- "properties": {
- "choice": c,
- "doc": {"type": "identity"}
- }
- }
- for c in choices
- ],
- "test": {
- "type": "boolean_expression",
- "expression": {
- "type": "property_path",
- "property_path": ["doc"] + path
- },
- "operator": "in_multi",
- "property_value": {"type": "property_name", "property_name": "choice"}
- }
- },
- "map_expression": sub_doc(path)
- }
- @property
- def source_doc_type(self):
- return 'XFormInstance'
- @property
- def uses_registry_data_source(self):
- return False
- @property
- @memoized
- def filter(self):
- return make_form_data_source_filter(
- self.source_xform.data_node.tag_xmlns, self.source_form.get_app().get_id)
- @property
- @memoized
- def data_source_properties(self):
- property_map = {
- 'username': _('User Name'),
- 'userID': _('User ID'),
- 'timeStart': _('Date Form Started'),
- 'timeEnd': _('Date Form Completed'),
- }
- properties = OrderedDict()
- questions = self.source_xform.get_questions([], exclude_select_with_itemsets=True)
- for prop in FORM_METADATA_PROPERTIES:
- question_type = prop[1]
- data_type = {
- "DateTime": "datetime",
- "Text": "string",
- }[question_type]
- properties[prop[0]] = DataSourceProperty(
- type=PROPERTY_TYPE_META,
- id=prop[0],
- text=property_map.get(prop[0], prop[0]),
- source=prop,
- data_types=[data_type]
- )
- for question in questions:
- if question['type'] == "DataBindOnly":
- data_types = ["string", "decimal", "datetime"]
- elif question['type'] in ("Int", "Double", "Long"):
- data_types = ["decimal"]
- else:
- data_types = ["string"]
- properties[question['value']] = DataSourceProperty(
- type=PROPERTY_TYPE_QUESTION,
- id=question['value'],
- text=question['label'],
- source=question,
- data_types=data_types,
- )
- if self.source_form.get_app().auto_gps_capture:
- properties['location'] = DataSourceProperty(
- type=PROPERTY_TYPE_META,
- id='location',
- text='location',
- source=(['location', '#text'], 'Text'),
- data_types=["string"],
- )
- return properties
- def _get_data_source_build_information(self):
- return DataSourceBuildInformation(
- source_id=self.source_id,
- app_id=self.app._id,
- app_version=self.app.version,
- )
- @property
- @memoized
- def data_source_name(self):
- today = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
- return "{} (v{}) {}".format(self.source_form.default_name(), self.app.version, today)
- class CaseDataSourceHelper(ManagedReportBuilderDataSourceHelper):
- """
- A ReportBuilderDataSourceInterface specifically for when source_type = 'case'.
- """
- @property
- def source_doc_type(self):
- return 'CommCareCase'
- @property
- @memoized
- def filter(self):
- return make_case_data_source_filter(self.source_id)
- def base_item_expression(self, is_multiselect_chart_report, multiselect_field=None):
- assert not is_multiselect_chart_report
- return {}
- @property
- @memoized
- def data_source_properties(self):
- property_map = {
- 'closed': _('Case Closed'),
- 'user_id': _('User ID Last Updating Case'),
- 'owner_name': _('Case Owner'),
- 'mobile worker': _('Mobile Worker Last Updating Case'),
- 'case_id': _('Case ID')
- }
- properties = OrderedDict()
- for property in self.case_properties:
- if property in DEFAULT_CASE_PROPERTY_DATATYPES:
- data_types = DEFAULT_CASE_PROPERTY_DATATYPES[property]
- else:
- data_types = ["string", "decimal", "datetime"]
- properties[property] = DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=property,
- text=property_map.get(property, property.replace('_', ' ')),
- source=property,
- data_types=data_types,
- )
- properties[COMPUTED_OWNER_NAME_PROPERTY_ID] = self._get_owner_name_pseudo_property()
- properties[COMPUTED_USER_NAME_PROPERTY_ID] = self._get_user_name_pseudo_property()
- if SHOW_IDS_IN_REPORT_BUILDER.enabled(self.domain):
- properties['case_id'] = self._get_case_id_pseudo_property()
- if SHOW_OWNER_LOCATION_PROPERTY_IN_REPORT_BUILDER.enabled(self.domain):
- properties[COMPUTED_OWNER_LOCATION_PROPERTY_ID] = self._get_owner_location_pseudo_property()
- properties[COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID] = \
- self._get_owner_location_with_descendants_pseudo_property()
- properties[COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID] = \
- self._get_owner_location_archived_with_descendants_pseudo_property()
- return properties
- @staticmethod
- def _get_case_id_pseudo_property():
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id='case_id',
- text=_('Case ID'),
- source='case_id',
- data_types=["string"],
- )
- @staticmethod
- def _get_owner_name_pseudo_property():
- # owner_name is a special pseudo-case property for which
- # the report builder will create a related_doc indicator based
- # on the owner_id of the case.
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=COMPUTED_OWNER_NAME_PROPERTY_ID,
- text=_('Case Owner'),
- source=COMPUTED_OWNER_NAME_PROPERTY_ID,
- data_types=["string"],
- )
- @classmethod
- def _get_owner_location_pseudo_property(cls):
- # owner_location is a special pseudo-case property for which
- # the report builder reference the owner_id, but treat it as a location
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=COMPUTED_OWNER_LOCATION_PROPERTY_ID,
- text=_('Case Owner (Location)'),
- source=COMPUTED_OWNER_LOCATION_PROPERTY_ID,
- data_types=["string"],
- )
- @classmethod
- def _get_owner_location_with_descendants_pseudo_property(cls):
- # similar to the location property but also include descendants
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID,
- text=_('Case Owner (Location w/ Descendants)'),
- source=COMPUTED_OWNER_LOCATION_WITH_DESENDANTS_PROPERTY_ID,
- data_types=["string"],
- )
- @classmethod
- def _get_owner_location_archived_with_descendants_pseudo_property(cls):
- # similar to the location property but also include descendants
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID,
- text=_('Case Owner (Location w/ Descendants and Archived Locations)'),
- source=COMPUTED_OWNER_LOCATION_ARCHIVED_WITH_DESCENDANTS_PROPERTY_ID,
- data_types=["string"],
- )
- @staticmethod
- def _get_user_name_pseudo_property():
- # user_name is a special pseudo case property for which
- # the report builder will create a related_doc indicator based on the
- # user_id of the case
- return DataSourceProperty(
- type=PROPERTY_TYPE_CASE_PROP,
- id=COMPUTED_USER_NAME_PROPERTY_ID,
- text=_('Mobile Worker Last Updating Case'),
- source=COMPUTED_USER_NAME_PROPERTY_ID,
- data_types=["string"],
- )
- class ApplicationCaseDataSourceHelper(CaseDataSourceHelper):
- def __init__(self, domain, app, source_type, source_id):
- self.app = app
- assert source_type == 'case'
- super().__init__(domain, source_type, source_id)
- prop_map = get_case_properties(
- self.app, [self.source_id], defaults=list(DEFAULT_CASE_PROPERTY_DATATYPES),
- include_parent_properties=True,
- )
- self.case_properties = sorted(set(prop_map[self.source_id]) | {'closed', 'closed_on'})
- def _get_data_source_build_information(self):
- return DataSourceBuildInformation(
- source_id=self.source_id,
- app_id=self.app._id,
- app_version=self.app.version,
- )
- @property
- def uses_registry_data_source(self):
- return False
- @property
- @memoized
- def data_source_name(self):
- today = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
- return "{} (v{}) {}".format(self.source_id, self.app.version, today)
- class RegistryCaseDataSourceHelper(CaseDataSourceHelper):
- def __init__(self, domain, registry_slug, source_type, source_id):
- assert source_type == 'case'
- self.registry_slug = registry_slug
- super().__init__(domain, source_type, source_id)
- registry_helper = DataRegistryHelper(self.domain, registry_slug=self.registry_slug)
- owning_domain = registry_helper.registry.domain
- prop_map = get_data_dict_props_by_case_type(owning_domain)
- self.case_properties = sorted(
- set(prop_map[self.source_id]) | {'closed', 'closed_on'}
- )
- def _get_data_source_build_information(self):
- return DataSourceBuildInformation(
- source_id=self.source_id,
- registry_slug=self.registry_slug,
- )
- @property
- def uses_registry_data_source(self):
- return True
- @property
- @memoized
- def data_source_name(self):
- today = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
- return "{} {} {}".format(self.source_id, self.registry_slug, today)
- @property
- def data_source_properties(self):
- properties = super().data_source_properties
- properties[COMMCARE_PROJECT] = DataSourceProperty(
- type=PROPERTY_TYPE_RAW,
- id=COMMCARE_PROJECT,
- text=_('CommCare Project'),
- source='commcare_project',
- data_types=["string"],
- )
- return properties
- def construct_data_source(self, table_id, **kwargs):
- return RegistryDataSourceConfiguration(
- domain=self.domain,
- table_id=table_id,
- registry_slug=self.registry_slug,
- **kwargs
- )
- def _get_datasource_default_columns(self):
- return {
- column.id
- for indicator in RegistryDataSourceConfiguration().default_indicators
- for column in indicator.get_columns()
- }
- def get_data_source_interface(domain, app, source_type, source_id, registry_slug):
- if registry_slug is not None and source_type == DATA_SOURCE_TYPE_CASE:
- return RegistryCaseDataSourceHelper(domain, registry_slug, source_type, source_id)
- if source_type in APP_DATA_SOURCE_TYPE_VALUES:
- helper = {
- DATA_SOURCE_TYPE_CASE: ApplicationCaseDataSourceHelper,
- DATA_SOURCE_TYPE_FORM: ApplicationFormDataSourceHelper
- }[source_type]
- return helper(domain, app, source_type, source_id)
- else:
- return UnmanagedDataSourceHelper(domain, app, source_type, source_id)
- class DataSourceForm(forms.Form):
- report_name = forms.CharField()
- def __init__(self, domain, max_allowed_reports, request_user, *args, **kwargs):
- super(DataSourceForm, self).__init__(*args, **kwargs)
- self.domain = domain
- self.max_allowed_reports = max_allowed_reports
- self.request_user = request_user
- self.registry_permission_checker = RegistryPermissionCheck(self.domain, self.request_user)
- # TODO: Map reports.
- self.app_source_helper = ApplicationDataSourceUIHelper(
- enable_raw=SHOW_RAW_DATA_SOURCES_IN_REPORT_BUILDER.enabled(self.domain),
- enable_registry=(DATA_REGISTRY_UCR.enabled(self.domain)
- and self.registry_permission_checker.can_view_some_data_registry_contents()),
- registry_permission_checker=self.registry_permission_checker
- )
- self.app_source_helper.bootstrap(self.domain)
- self.fields.update(self.app_source_helper.get_fields())
- self.helper = FormHelper()
- self.helper.form_class = "form form-horizontal"
- self.helper.form_id = "report-builder-form"
- self.helper.label_class = 'col-sm-3 col-md-2 col-lg-2'
- self.helper.field_class = 'col-sm-9 col-md-8 col-lg-6'
- self.helper.layout = crispy.Layout(
- crispy.Fieldset(
- _('Report'),
- hqcrispy.FieldWithHelpBubble(
- 'report_name',
- help_bubble_text=_(
- 'Web users will see this name in the "Reports" section of CommCareHQ and can click to '
- 'view the report'
- )
- )
- ),
- self.get_data_layout(),
- hqcrispy.FormActions(
- StrictButton(
- _('Next'),
- type="submit",
- css_id='js-next-data-source',
- css_class="btn-primary",
- )
- ),
- )
- def get_data_layout(self):
- if not (DATA_REGISTRY_UCR.enabled(self.domain)
- and self.registry_permission_checker.can_view_some_data_registry_contents()):
- return crispy.Fieldset(
- _('Data'), *self.app_source_helper.get_crispy_fields(),
- )
- else:
- help_texts = self.app_source_helper.get_crispy_filed_help_texts()
- return crispy.Fieldset(
- _('Data'),
- hqcrispy.FieldWithHelpBubble('source_type', help_bubble_text=help_texts['source_type']),
- crispy.Div(
- crispy.HTML('<input type="radio" name="project_data" id="one_project" '
- 'value="isDataFromOneProject" data-bind="checked: isDataFromOneProject,'
- ' checkedValue: \'true\'" class="project_data-option"/>'
- '<label for="one_project" class="project_data-label">%s</label>'
- % _("Data From My Project Space")),
- crispy.Div(
- hqcrispy.FieldWithHelpBubble('application', help_bubble_text=help_texts['application']),
- style="padding-left: 50px;"
- ),
- crispy.HTML('<input type="radio" name="project_data" id="many_projects" '
- 'value="isDataFromOneProject" data-bind="checked: isDataFromOneProject, '
- 'checkedValue: \'false\'" class="project_data-option"/><label '
- 'for="many_projects" class="project_data-label">%s</label>'
- % _("Data From My Project Space And Others")),
- crispy.Div(
- hqcrispy.FieldWithHelpBubble('registry_slug', help_bubble_text=help_texts['registry_slug']),
- style="padding-left: 50px;"
- ),
- ),
- hqcrispy.FieldWithHelpBubble('source', help_bubble_text=help_texts['source']),
- )
- @property
- def sources_map(self):
- return self.app_source_helper.all_sources
- @property
- def dropdown_map(self):
- return self.app_source_helper.app_and_registry_sources
- def get_selected_source(self):
- return self.app_source_helper.get_app_source(self.cleaned_data)
- def clean(self):
- """
- Raise a validation error if there are already 5 data sources and this
- report won't be able to use one of the existing ones.
- """
- cleaned_data = super(DataSourceForm, self).clean()
- existing_reports = ReportConfiguration.by_domain(self.domain)
- builder_reports = [report for report in existing_reports if report.report_meta.created_by_builder]
- if has_report_builder_access(self.domain) and len(builder_reports) >= self.max_allowed_reports:
- # Don't show the warning when domain does not have report buidler access, because this is just a
- # preview and the report will not be saved.
- raise forms.ValidationError(_(
- "Too many reports!\n"
- "Creating this report would cause you to go over the maximum "
- "number of report builder reports allowed in this domain. Your "
- "limit is {number}. "
- "To continue, delete another report and try again. "
- ).format(number=self.max_allowed_reports))
- return cleaned_data
- _shared_properties = ['exists_in_current_version', 'display_text', 'property', 'data_source_field']
- UserFilterViewModel = namedtuple("UserFilterViewModel", _shared_properties + ['format'])
- DefaultFilterViewModel = namedtuple("DefaultFilterViewModel",
- _shared_properties + ['format', 'pre_value', 'pre_operator'])
- ColumnViewModel = namedtuple("ColumnViewModel", _shared_properties + ['calculation'])
- class ConfigureNewReportBase(forms.Form):
- user_filters = FilterField(required=False)
- default_filters = FilterField(required=False)
- report_title = forms.CharField(widget=forms.HiddenInput, required=False)
- report_description = forms.CharField(widget=forms.HiddenInput, required=False)
- def __init__(self, domain, report_name, app_id, source_type, report_source_id, existing_report=None, registry_slug=None,
- *args, **kwargs):
- """
- This form can be used to create a new ReportConfiguration, or to modify
- an existing one if existing_report is set.
- """
- super(ConfigureNewReportBase, self).__init__(*args, **kwargs)
- self.existing_report = existing_report
- self.domain = domain
- if self.existing_report:
- self._bootstrap(self.existing_report)
- else:
- self.registry_slug = registry_slug
- self.report_name = report_name
- assert source_type in REPORT_BUILDER_DATA_SOURCE_TYPE_VALUES
- self.source_type = source_type
- self.report_source_id = report_source_id
- self.app = Application.get(app_id) if app_id else None
- if self.app:
- assert self.domain == self.app.domain
- self.ds_builder = get_data_source_interface(
- self.domain, self.app, self.source_type, self.report_source_id, self.registry_slug
- )
- self.report_column_options = self.ds_builder.report_column_options
- self.data_source_properties = self.ds_builder.data_source_properties
- self._report_columns_by_column_id = {}
- for column in self.report_column_options.values():
- for agg in column.aggregation_options:
- indicators = column.get_indicators(agg)
- for i in indicators:
- self._report_columns_by_column_id[i['column_id']] = column
- def _bootstrap(self, existing_report):
- """
- Use an existing report to initialize some of the instance variables of this
- form. This method is used when editing an existing report.
- """
- self.report_name = existing_report.title
- self.source_type = get_source_type_from_report_config(existing_report)
- assert self.domain == existing_report.domain
- if self.source_type in APP_DATA_SOURCE_TYPE_VALUES:
- self.report_source_id = existing_report.config.meta.build.source_id
- app_id = existing_report.config.meta.build.app_id
- self.registry_slug = existing_report.config.meta.build.registry_slug
- self.app = None
- if app_id:
- self.app = Application.get(app_id)
- elif not self.registry_slug:
- raise BadBuilderConfigError(DATA_SOURCE_MISSING_APP_ERROR_MESSAGE)
- else:
- assert self.source_type == DATA_SOURCE_TYPE_RAW
- self.report_source_id = existing_report.config_id
- self.app = self.registry_slug = None
- @property
- def _configured_columns(self):
- """
- To be used by ManagedReportBuilderDataSourceHelper.indicators()
- """
- configured_columns = self.cleaned_data['columns']
- location = self.cleaned_data.get("location")
- if location and all(location != c.get('property')
- for c in configured_columns):
- configured_columns += [{
- "property": location,
- "calculation": UI_AGG_GROUP_BY # Not aggregated
- }]
- return configured_columns
- def _get_data_source_configuration_kwargs(self):
- filters = self.cleaned_data['user_filters'] + self.cleaned_data['default_filters']
- ms_field = self._report_aggregation_cols[0] if self._is_multiselect_chart_report else None
- return self.ds_builder.get_datasource_constructor_kwargs(self._configured_columns,
- filters,
- self._is_multiselect_chart_report,
- ms_field)
- def _build_data_source(self):
- data_source_config = self.ds_builder.construct_data_source(
- # The uuid gets truncated, so it's not really universally unique.
- table_id=clean_table_name(self.domain, str(uuid.uuid4().hex)),
- **self._get_data_source_configuration_kwargs()
- )
- data_source_config.validate()
- data_source_config.save()
- tasks.rebuild_indicators.delay(data_source_config._id, source="report_builder", domain=data_source_config.domain)
- return data_source_config._id
- def update_report(self):
- self._update_data_source_if_necessary()
- self.existing_report.aggregation_columns = self._report_aggregation_cols
- self.existing_report.columns = self._get_report_columns()
- self.existing_report.filters = self._report_filters
- self.existing_report.configured_charts = self._report_charts
- self.existing_report.title = self.cleaned_data['report_title'] or _("Report Builder Report")
- self.existing_report.description = self.cleaned_data['report_description']
- self.existing_report.validate()
- self.existing_report.save()
- DomainAuditRecordEntry.update_calculations(self.domain, 'cp_n_reports_edited')
- return self.existing_report
- def _update_data_source_if_necessary(self):
- if self.ds_builder.uses_managed_data_source:
- data_source = get_ucr_datasource_config_by_id(self.existing_report.config_id)
- if data_source.get_report_count() > 1:
- # If another report is pointing at this data source, create a new
- # data source for this report so that we can change the indicators
- # without worrying about breaking another report.
- data_source_config_id = self._build_data_source()
- self.existing_report.config_id = data_source_config_id
- else:
- indicators = self.ds_builder.indicators(
- self._configured_columns,
- self.cleaned_data['user_filters'] + self.cleaned_data['default_filters'],
- )
- if data_source.configured_indicators != indicators:
- for property_name, value in self._get_data_source_configuration_kwargs().items():
- setattr(data_source, propert