/corehq/apps/app_manager/util.py
Python | 731 lines | 664 code | 40 blank | 27 comment | 26 complexity | 7355e740854dbb717cd7787ca173bb94 MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-2.1
- import json
- import logging
- import os
- import re
- import uuid
- from collections import OrderedDict, namedtuple
- from copy import deepcopy
- from django.core.cache import cache
- from django.db.models import Max
- from django.http import Http404
- from django.urls import reverse
- from django.utils.translation import gettext as _
- import yaml
- from couchdbkit import ResourceNotFound
- from couchdbkit.exceptions import DocTypeError
- from dimagi.utils.couch import CriticalSection
- from corehq import toggles
- from corehq.apps.app_manager.const import (
- AUTO_SELECT_USERCASE,
- REGISTRY_WORKFLOW_LOAD_CASE,
- REGISTRY_WORKFLOW_SMART_LINK,
- USERCASE_ID,
- USERCASE_PREFIX,
- USERCASE_TYPE,
- )
- from corehq.apps.app_manager.dbaccessors import get_app, get_apps_in_domain
- from corehq.apps.app_manager.exceptions import (
- AppManagerException,
- PracticeUserException,
- SuiteError,
- SuiteValidationError,
- XFormException,
- )
- from corehq.apps.app_manager.tasks import create_usercases
- from corehq.apps.app_manager.xform import XForm, parse_xml
- from corehq.apps.app_manager.xpath import UsercaseXPath
- from corehq.apps.builds.models import CommCareBuildConfig
- from corehq.apps.domain.models import Domain
- from corehq.apps.locations.models import SQLLocation
- from corehq.apps.users.models import CommCareUser
- from corehq.util.quickcache import quickcache
- from corehq.util.soft_assert import soft_assert
- logger = logging.getLogger(__name__)
- CASE_XPATH_SUBSTRING_MATCHES = [
- "instance('casedb')",
- 'session/data/case_id',
- "#case",
- "#parent",
- "#host",
- ]
- USERCASE_XPATH_SUBSTRING_MATCHES = [
- "#user",
- UsercaseXPath().case(),
- ]
- def app_doc_types():
- from corehq.apps.app_manager.models import Application, RemoteApp, LinkedApplication
- return {
- 'Application': Application,
- 'Application-Deleted': Application,
- 'RemoteApp': RemoteApp,
- 'RemoteApp-Deleted': RemoteApp,
- 'LinkedApplication': LinkedApplication,
- 'LinkedApplication-Deleted': LinkedApplication
- }
- def is_linked_app(app_or_doc, include_deleted=False):
- return _get_doc_type(app_or_doc) in ('LinkedApplication', 'LinkedApplication-Deleted')
- def is_remote_app(app_or_doc, include_deleted=False):
- return _get_doc_type(app_or_doc) in ('RemoteApp', 'RemoteApp-Deleted')
- def _get_doc_type(app_or_doc):
- if hasattr(app_or_doc, 'doc_type'):
- doc_type = app_or_doc.doc_type
- elif 'doc_type' in app_or_doc:
- doc_type = app_or_doc['doc_type']
- assert doc_type
- return doc_type
- def _prepare_xpath_for_validation(xpath):
- prepared_xpath = xpath.lower()
- prepared_xpath = prepared_xpath.replace('"', "'")
- prepared_xpath = re.compile(r'\s').sub('', prepared_xpath)
- return prepared_xpath
- def _check_xpath_for_matches(xpath, substring_matches=None, pattern_matches=None):
- prepared_xpath = _prepare_xpath_for_validation(xpath)
- substring_matches = substring_matches or []
- pattern_matches = pattern_matches or []
- return any([
- re.compile(pattern).search(prepared_xpath) for pattern in pattern_matches
- ] + [
- substring in prepared_xpath for substring in substring_matches
- ])
- def xpath_references_case(xpath):
- # We want to determine here if the xpath references any cases other
- # than the user case. To determine if the xpath references the user
- # case, see xpath_references_usercase()
- # Assumes xpath has already been dot interpolated as needed.
- for substring in USERCASE_XPATH_SUBSTRING_MATCHES:
- xpath = xpath.replace(substring, '')
- return _check_xpath_for_matches(
- xpath,
- substring_matches=CASE_XPATH_SUBSTRING_MATCHES,
- )
- def xpath_references_usercase(xpath):
- # Assumes xpath has already been dot interpolated as needed.
- return _check_xpath_for_matches(
- xpath,
- substring_matches=USERCASE_XPATH_SUBSTRING_MATCHES,
- )
- def split_path(path):
- path_parts = path.split('/')
- name = path_parts.pop(-1)
- path = '/'.join(path_parts)
- return path, name
- def first_elem(elem_list):
- return elem_list[0] if elem_list else None
- def generate_xmlns():
- return str(uuid.uuid4()).upper()
- def save_xform(app, form, xml):
- def change_xmlns(xform, old_xmlns, new_xmlns):
- data = xform.data_node.render().decode('utf-8')
- data = data.replace(old_xmlns, new_xmlns, 1)
- xform.instance_node.remove(xform.data_node.xml)
- xform.instance_node.append(parse_xml(data))
- return xform.render()
- try:
- xform = XForm(xml, domain=app.domain)
- except XFormException:
- pass
- else:
- GENERIC_XMLNS = "http://www.w3.org/2002/xforms"
- uid = generate_xmlns()
- tag_xmlns = xform.data_node.tag_xmlns
- new_xmlns = form.xmlns or "http://openrosa.org/formdesigner/%s" % uid
- if not tag_xmlns or tag_xmlns == GENERIC_XMLNS: # no xmlns
- xml = change_xmlns(xform, GENERIC_XMLNS, new_xmlns)
- else:
- forms = [form_
- for form_ in app.get_xmlns_map().get(tag_xmlns, [])
- if form_.form_type != 'shadow_form']
- if len(forms) > 1 or (len(forms) == 1 and forms[0] is not form):
- if new_xmlns == tag_xmlns:
- new_xmlns = "http://openrosa.org/formdesigner/%s" % uid
- # form most likely created by app.copy_form(...)
- # or form is being updated with source copied from other form
- xml = change_xmlns(xform, tag_xmlns, new_xmlns)
- form.source = xml.decode('utf-8')
- if form.is_registration_form():
- # For registration forms, assume that the first question is the
- # case name unless something else has been specified
- questions = form.get_questions([app.default_language])
- if hasattr(form.actions, 'open_case'):
- path = form.actions.open_case.name_update.question_path
- if path:
- name_questions = [q for q in questions if q['value'] == path]
- if not len(name_questions):
- path = None
- if not path and len(questions):
- form.actions.open_case.name_update.question_path = questions[0]['value']
- return xml
- CASE_TYPE_REGEX = r'^[\w-]+$'
- _case_type_regex = re.compile(CASE_TYPE_REGEX)
- def is_valid_case_type(case_type, module):
- """
- >>> from corehq.apps.app_manager.models import Module, AdvancedModule
- >>> is_valid_case_type('foo', Module())
- True
- >>> is_valid_case_type('foo-bar', Module())
- True
- >>> is_valid_case_type('foo bar', Module())
- False
- >>> is_valid_case_type('', Module())
- False
- >>> is_valid_case_type(None, Module())
- False
- >>> is_valid_case_type('commcare-user', Module())
- False
- >>> is_valid_case_type('commcare-user', AdvancedModule())
- True
- """
- from corehq.apps.app_manager.models import AdvancedModule
- matches_regex = bool(_case_type_regex.match(case_type or ''))
- prevent_usercase_type = (case_type != USERCASE_TYPE or isinstance(module, AdvancedModule))
- return matches_regex and prevent_usercase_type
- def module_case_hierarchy_has_circular_reference(module):
- from corehq.apps.app_manager.suite_xml.utils import get_select_chain
- try:
- get_select_chain(module.get_app(), module)
- return False
- except SuiteValidationError:
- return True
- def is_usercase_in_use(domain_name):
- domain_obj = Domain.get_by_name(domain_name) if domain_name else None
- return domain_obj and domain_obj.usercase_enabled
- def get_settings_values(app):
- try:
- profile = app.profile
- except AttributeError:
- profile = {}
- hq_settings = dict([
- (attr, app[attr])
- for attr in app.properties() if not hasattr(app[attr], 'pop')
- ])
- if getattr(app, 'use_custom_suite', False):
- hq_settings.update({'custom_suite': getattr(app, 'custom_suite', None)})
- hq_settings['build_spec'] = app.build_spec.to_string()
- # the admin_password hash shouldn't be sent to the client
- hq_settings.pop('admin_password', None)
- # convert int to string
- hq_settings['mobile_ucr_restore_version'] = str(hq_settings.get('mobile_ucr_restore_version', '1.0'))
- domain_obj = Domain.get_by_name(app.domain)
- return {
- 'properties': profile.get('properties', {}),
- 'features': profile.get('features', {}),
- 'hq': hq_settings,
- '$parent': {
- 'doc_type': app.get_doc_type(),
- '_id': app.get_id,
- 'domain': app.domain,
- 'commtrack_enabled': domain_obj.commtrack_enabled,
- }
- }
- def add_odk_profile_after_build(app_build):
- """caller must save"""
- profile = app_build.create_profile(is_odk=True)
- app_build.lazy_put_attachment(profile, 'files/profile.ccpr')
- # hack this in for records
- app_build.odk_profile_created_after_build = True
- def create_temp_sort_column(sort_element, order):
- """
- Used to create a column for the sort only properties to
- add the field to the list of properties and app strings but
- not persist anything to the detail data.
- """
- from corehq.apps.app_manager.models import DetailColumn
- col = DetailColumn(
- model='case',
- field=sort_element.field,
- format='invisible',
- header=sort_element.display,
- )
- col._i = order
- return col
- def get_correct_app_class(doc):
- try:
- return app_doc_types()[doc['doc_type']]
- except KeyError:
- raise DocTypeError(doc['doc_type'])
- def languages_mapping():
- mapping = cache.get('__languages_mapping')
- if not mapping:
- with open('submodules/langcodes/langs.json', encoding='utf-8') as langs_file:
- lang_data = json.load(langs_file)
- mapping = dict([(l["two"], l["names"]) for l in lang_data])
- mapping["default"] = ["Default Language"]
- cache.set('__languages_mapping', mapping, 12*60*60)
- return mapping
- def version_key(ver):
- """
- A key function that takes a version and returns a numeric value that can
- be used for sorting
- >>> version_key('2')
- 2000000
- >>> version_key('2.9')
- 2009000
- >>> version_key('2.10')
- 2010000
- >>> version_key('2.9.1')
- 2009001
- >>> version_key('2.9.1.1')
- 2009001
- >>> version_key('2.9B')
- Traceback (most recent call last):
- ...
- ValueError: invalid literal for int() with base 10: '9B'
- """
- padded = ver + '.0.0'
- values = padded.split('.')
- return int(values[0]) * 1000000 + int(values[1]) * 1000 + int(values[2])
- def get_commcare_versions(request_user):
- versions = [i.version for i in get_commcare_builds(request_user)]
- return sorted(versions, key=version_key)
- def get_commcare_builds(request_user):
- can_view_superuser_builds = (request_user.is_superuser
- or toggles.IS_CONTRACTOR.enabled(request_user.username))
- return [
- i.build
- for i in CommCareBuildConfig.fetch().menu
- if can_view_superuser_builds or not i.superuser_only
- ]
- def actions_use_usercase(actions):
- return (('usercase_update' in actions and actions['usercase_update'].update) or
- ('usercase_preload' in actions and actions['usercase_preload'].preload))
- def advanced_actions_use_usercase(actions):
- return any(c.auto_select and c.auto_select.mode == AUTO_SELECT_USERCASE for c in actions.load_update_cases)
- def enable_usercase(domain_name):
- with CriticalSection(['enable_usercase_' + domain_name]):
- domain_obj = Domain.get_by_name(domain_name, strict=True)
- if not domain_obj: # copying domains passes in an id before name is saved
- domain_obj = Domain.get(domain_name)
- if not domain_obj.usercase_enabled:
- domain_obj.usercase_enabled = True
- domain_obj.save()
- create_usercases.delay(domain_name)
- def prefix_usercase_properties(properties):
- return {'{}{}'.format(USERCASE_PREFIX, prop) for prop in properties}
- def module_offers_registry_search(module):
- return (
- module_offers_search(module)
- and module.get_app().supports_data_registry
- and module.search_config.data_registry
- )
- def module_loads_registry_case(module):
- return (
- module_offers_registry_search(module)
- and module.search_config.data_registry_workflow == REGISTRY_WORKFLOW_LOAD_CASE
- )
- def module_uses_smart_links(module):
- return (
- module_offers_registry_search(module)
- and module.search_config.data_registry_workflow == REGISTRY_WORKFLOW_SMART_LINK
- )
- def module_offers_search(module):
- from corehq.apps.app_manager.models import AdvancedModule, Module, ShadowModule
- return (
- isinstance(module, (Module, AdvancedModule, ShadowModule)) and
- module.search_config and
- (module.search_config.properties or
- module.search_config.default_properties)
- )
- def module_uses_inline_search(module):
- """In 'inline search' mode the query and post are added to the form entry directly instead
- of creating a separate RemoteRequest entry."""
- return (
- module_offers_search(module)
- and module.search_config.inline_search
- and module.search_config.auto_launch
- )
- def get_cloudcare_session_data(domain_name, form, couch_user):
- from corehq.apps.app_manager.suite_xml.sections.entries import EntriesHelper
- datums = EntriesHelper.get_new_case_id_datums_meta(form)
- session_data = {datum.id: uuid.uuid4().hex for datum in datums}
- if couch_user.doc_type == 'CommCareUser': # smsforms.app.start_session could pass a CommCareCase
- try:
- extra_datums = EntriesHelper.get_extra_case_id_datums(form)
- except SuiteError as err:
- _assert = soft_assert(['nhooper_at_dimagi_dot_com'.replace('_at_', '@').replace('_dot_', '.')])
- _assert(False, 'Domain "%s": %s' % (domain_name, err))
- else:
- if EntriesHelper.any_usercase_datums(extra_datums):
- usercase_id = couch_user.get_usercase_id()
- if usercase_id:
- session_data[USERCASE_ID] = usercase_id
- return session_data
- def update_form_unique_ids(app_source, ids_map, update_all=True):
- """
- Accepts an ids_map translating IDs in app_source to the desired replacement
- ID. Form IDs not present in ids_map will be given new random UUIDs.
- """
- from corehq.apps.app_manager.models import form_id_references, jsonpath_update
- app_source = deepcopy(app_source)
- attachments = app_source['_attachments']
- def change_form_unique_id(form, old_id, new_id):
- form['unique_id'] = new_id
- if f"{old_id}.xml" in attachments:
- attachments[f"{new_id}.xml"] = attachments.pop(f"{old_id}.xml")
- # once Application.wrap includes deleting user_registration
- # we can remove this
- if 'user_registration' in app_source:
- del app_source['user_registration']
- new_ids_by_old = {}
- for m, module in enumerate(app_source['modules']):
- for f, form in enumerate(module['forms']):
- old_id = form['unique_id']
- if update_all or old_id in ids_map:
- new_id = ids_map.get(old_id, uuid.uuid4().hex)
- new_ids_by_old[old_id] = new_id
- change_form_unique_id(form, old_id, new_id)
- for reference_path in form_id_references:
- for reference in reference_path.find(app_source):
- if reference.value in new_ids_by_old:
- jsonpath_update(reference, new_ids_by_old[reference.value])
- return app_source
- def update_report_module_ids(app_source):
- """Make new report UUIDs so they stay unique
- Otherwise there would be multiple reports in the restore with the same UUID
- Set the report slug to the old UUID so any xpath expressions referencing
- the report by ID continue to work, if only in mobile UCR v2
- """
- app_source = deepcopy(app_source)
- for module in app_source['modules']:
- if module['module_type'] == 'report':
- for config in module['report_configs']:
- if not config.get('report_slug'):
- config['report_slug'] = config['uuid']
- config['uuid'] = uuid.uuid4().hex
- return app_source
- def _app_callout_templates():
- """Load app callout templates from config file on disk
- Generator function defers file access until needed, acts like a
- constant thereafter.
- """
- path = os.path.join(
- os.path.dirname(__file__),
- 'static', 'app_manager', 'json', 'vellum-app-callout-templates.yml'
- )
- if os.path.exists(path):
- with open(path, encoding='utf-8') as f:
- data = yaml.safe_load(f)
- else:
- logger.info("not found: %s", path)
- data = []
- while True:
- yield data
- app_callout_templates = _app_callout_templates()
- def purge_report_from_mobile_ucr(report_config):
- """
- Called when a report is deleted, this will remove any references to it in
- mobile UCR modules.
- """
- if not toggles.MOBILE_UCR.enabled(report_config.domain):
- return False
- did_purge_something = False
- for app in get_apps_in_domain(report_config.domain):
- save_app = False
- for module in app.modules:
- if module.module_type == 'report':
- valid_report_configs = [
- app_config for app_config in module.report_configs
- if app_config.report_id != report_config._id
- ]
- if len(valid_report_configs) != len(module.report_configs):
- module.report_configs = valid_report_configs
- save_app = True
- if save_app:
- app.save()
- did_purge_something = True
- return did_purge_something
- SortOnlyElement = namedtuple("SortOnlyElement", "field, sort_element, order")
- def get_sort_and_sort_only_columns(detail_columns, sort_elements):
- """
- extracts out info about columns that are added as only sort fields and columns added as both
- sort and display fields
- """
- sort_elements = OrderedDict((s.field, (s, i + 1)) for i, s in enumerate(sort_elements))
- sort_columns = {}
- for column in detail_columns:
- sort_element, order = sort_elements.pop(column.field, (None, None))
- if sort_element:
- sort_columns[column.field] = (sort_element, order)
- sort_only_elements = [
- SortOnlyElement(field, element, element_order)
- for field, (element, element_order) in sort_elements.items()
- ]
- return sort_only_elements, sort_columns
- def get_and_assert_practice_user_in_domain(practice_user_id, domain):
- # raises PracticeUserException if CommCareUser with practice_user_id is not a practice mode user
- # or if user doesn't belong to domain
- try:
- user = CommCareUser.get(practice_user_id)
- if not user.domain == domain:
- raise ResourceNotFound
- except ResourceNotFound:
- raise PracticeUserException(
- _("Practice User with id {id} not found, please make sure you have not deleted this user").format(
- id=practice_user_id)
- )
- if not user.is_demo_user:
- raise PracticeUserException(
- _("User {username} is not a practice user, please turn on practice mode for this user").format(
- username=user.username)
- )
- if user.is_deleted():
- raise PracticeUserException(
- _("User {username} has been deleted, you can't use that user as practice user").format(
- username=user.username)
- )
- if not user.is_active:
- raise PracticeUserException(
- _("User {username} has been deactivated, you can't use that user as practice user").format(
- username=user.username)
- )
- return user
- def get_form_source_download_url(xform):
- """Returns the download url for the form source for a submitted XForm
- """
- if not xform.build_id:
- return None
- try:
- app = get_app(xform.domain, xform.build_id)
- except Http404:
- return None
- if app.is_remote_app():
- return None
- try:
- form = app.get_forms_by_xmlns(xform.xmlns)[0]
- except IndexError:
- return None
- return reverse("app_download_file", args=[
- xform.domain,
- xform.build_id,
- app.get_form_filename(module=form.get_module(), form=form),
- ])
- @quickcache(['domain', 'profile_id'], timeout=24 * 60 * 60)
- def get_latest_enabled_build_for_profile(domain, profile_id):
- from corehq.apps.app_manager.models import LatestEnabledBuildProfiles
- latest_enabled_build = (LatestEnabledBuildProfiles.objects.
- filter(build_profile_id=profile_id, active=True)
- .order_by('-version')
- .first())
- if latest_enabled_build:
- return get_app(domain, latest_enabled_build.build_id)
- @quickcache(['domain', 'location_id', 'app_id'], timeout=24 * 60 * 60)
- def get_latest_app_release_by_location(domain, location_id, app_id):
- """
- for a location search for enabled app releases for all parent locations.
- Child location's setting takes precedence over parent
- """
- from corehq.apps.app_manager.models import AppReleaseByLocation
- location = SQLLocation.active_objects.get(location_id=location_id)
- location_and_ancestor_ids = location.get_ancestors(include_self=True).values_list(
- 'location_id', flat=True).reverse()
- # get all active enabled releases and order by version desc to get one with the highest version in the end
- # for a location. Do not use the first object itself in order to respect the location hierarchy and use
- # the closest location to determine the valid active release
- latest_enabled_releases = {
- release.location_id: release.build_id
- for release in
- AppReleaseByLocation.objects.filter(
- location_id__in=location_and_ancestor_ids, app_id=app_id, domain=domain, active=True).order_by(
- 'version')
- }
- for loc_id in location_and_ancestor_ids:
- build_id = latest_enabled_releases.get(loc_id)
- if build_id:
- return get_app(domain, build_id)
- def expire_get_latest_app_release_by_location_cache(app_release_by_location):
- """
- expire cache for the location and its descendants for the app corresponding to this enabled app release
- why? : Latest enabled release for a location is dependent on restrictions added for
- itself and its ancestors. Hence we expire the cache for location and its descendants for which the
- latest enabled release would depend on this location
- """
- location = SQLLocation.active_objects.get(location_id=app_release_by_location.location_id)
- location_and_descendants = location.get_descendants(include_self=True)
- for loc in location_and_descendants:
- get_latest_app_release_by_location.clear(app_release_by_location.domain, loc.location_id,
- app_release_by_location.app_id)
- @quickcache(['app_id'], timeout=24 * 60 * 60)
- def get_latest_enabled_versions_per_profile(app_id):
- from corehq.apps.app_manager.models import LatestEnabledBuildProfiles
- # a dict with each profile id mapped to its latest enabled version number, if present
- return {
- build_profile['build_profile_id']: build_profile['version__max']
- for build_profile in
- LatestEnabledBuildProfiles.objects.filter(app_id=app_id, active=True).values('build_profile_id').annotate(
- Max('version'))
- }
- def get_app_id_from_form_unique_id(domain, form_unique_id):
- """
- Do not use. This is here to support migrations and temporary cose for *removing*
- the constraint that form ids be lgobally unique. It will stop working as more
- duplicated form unique ids appear.
- """
- return _get_app_ids_by_form_unique_id(domain).get(form_unique_id)
- @quickcache(['domain'], timeout=1 * 60 * 60)
- def _get_app_ids_by_form_unique_id(domain):
- apps = get_apps_in_domain(domain, include_remote=False)
- app_ids = {}
- for app in apps:
- for module in app.modules:
- for form in module.get_forms():
- if form.unique_id in app_ids:
- raise AppManagerException("Could not identify app for form {}".format(form.unique_id))
- app_ids[form.unique_id] = app.get_id
- return app_ids
- def extract_instance_id_from_nodeset_ref(nodeset):
- # note: for simplicity, this only returns the first instance ref in the event there are multiple.
- # if that's ever a problem this method could be changed in the future to return a list
- if nodeset:
- matches = re.findall(r"instance\('(.*?)'\)", nodeset)
- return matches[0] if matches else None
- def wrap_transition_from_old_update_case_action(properties_dict):
- """
- This function assists wrap functions for changes to the FormActions and AdvancedFormActions models.
- A modification of UpdateCaseAction to use a ConditionalCaseUpdate instead of a simple question path
- was part of these changes. It also used as part of a follow-up migration.
- """
- if(properties_dict):
- first_prop_value = list(properties_dict.values())[0]
- # If the dict just holds question paths (strings) as values we want to translate the old
- # type of UpdateCaseAction model to the new.
- if isinstance(first_prop_value, str):
- new_dict_values = {}
- for case_property, question_path in properties_dict.items():
- new_dict_values[case_property] = {
- 'question_path': question_path
- }
- return new_dict_values
- return properties_dict