/ion/services/ans/workflow_management_service.py

https://github.com/dstuebe/coi-services · Python · 252 lines · 115 code · 68 blank · 69 comment · 28 complexity · cf188c3a294f96e89d4a879c91f978a4 MD5 · raw file

  1. #!/usr/bin/env python
  2. __author__ = 'Stephen P. Henrie'
  3. __license__ = 'Apache 2.0'
  4. from interface.services.ans.iworkflow_management_service import BaseWorkflowManagementService
  5. from pyon.util.containers import is_basic_identifier
  6. from pyon.core.exception import BadRequest, NotFound, Inconsistent
  7. from pyon.public import Container, log, IonObject, RT,PRED, OT
  8. class WorkflowManagementService(BaseWorkflowManagementService):
  9. """
  10. The Workflow Management Service provides support for the definition, integration and enactment of
  11. defined workflows. Capabilities to enable the definition, instantiation, scheduling and execution
  12. control of developer provided workflows; specifically for the visualization of data products
  13. """
  14. def create_workflow_definition(self, workflow_definition=None):
  15. """Creates a Workflow Definition resource which specifies the steps involved in a workflow process.
  16. @param workflow_definition WorkflowDefinition
  17. @retval workflow_definition_id str
  18. @throws BadRequest if object passed has _id or _rev attribute
  19. """
  20. if not is_basic_identifier(workflow_definition.name):
  21. raise BadRequest("The workflow definition name '%s' can only contain alphanumeric and underscore characters" % workflow_definition.name)
  22. workflow_definition_id, version = self.clients.resource_registry.create(workflow_definition)
  23. workflow_definition = self.read_workflow_definition(workflow_definition_id)
  24. self._update_workflow_associations(workflow_definition)
  25. return workflow_definition_id
  26. def update_workflow_definition(self, workflow_definition=None):
  27. """Updates an existing Workflow Definition resource.
  28. @param workflow_definition WorkflowDefinition
  29. @throws BadRequest if object does not have _id or _rev attribute
  30. @throws NotFound object with specified id does not exist
  31. @throws Conflict object not based on latest persisted object version
  32. """
  33. if not is_basic_identifier(workflow_definition.name):
  34. raise BadRequest("The workflow definition name '%s' can only contain alphanumeric and underscore characters" % workflow_definition.name)
  35. self.clients.resource_registry.update(workflow_definition)
  36. self._update_workflow_associations(workflow_definition)
  37. def _delete_workflow_associations(self, workflow_definition_id):
  38. #Remove and existing associations
  39. aid_list = self.clients.resource_registry.find_associations(workflow_definition_id, PRED.hasDataProcessDefinition)
  40. for aid in aid_list:
  41. self.clients.resource_registry.delete_association(aid)
  42. def _update_workflow_associations(self, workflow_definition):
  43. #Remove and existing associations
  44. self._delete_workflow_associations(workflow_definition._id)
  45. #For each Data Process workflow step, create the appropriate associations
  46. for wf_step in workflow_definition.workflow_steps:
  47. if wf_step.type_ == OT.DataProcessWorkflowStep:
  48. self.clients.resource_registry.create_association(workflow_definition._id, PRED.hasDataProcessDefinition, wf_step.data_process_definition_id)
  49. def read_workflow_definition(self, workflow_definition_id=''):
  50. """Returns an existing Workflow Definition resource.
  51. @param workflow_definition_id str
  52. @retval workflow_definition WorkflowDefinition
  53. @throws BadRequest if any of the required parameters are not set
  54. @throws NotFound object with specified id does not exist
  55. """
  56. if not workflow_definition_id:
  57. raise BadRequest("The workflow_definition_id parameter is missing")
  58. workflow_definition = self.clients.resource_registry.read(workflow_definition_id)
  59. if not workflow_definition:
  60. raise NotFound("workflow_definition_id %s does not exist" % workflow_definition_id)
  61. return workflow_definition
  62. def delete_workflow_definition(self, workflow_definition_id=''):
  63. """Deletes an existing Workflow Definition resource.
  64. @param workflow_definition_id str
  65. @throws BadRequest if any of the required parameters are not set
  66. @throws NotFound object with specified id does not exist
  67. """
  68. if not workflow_definition_id:
  69. raise BadRequest("The workflow_definition_id parameter is missing")
  70. workflow_definition = self.clients.resource_registry.read(workflow_definition_id)
  71. if not workflow_definition:
  72. raise NotFound("workflow_definition_id %s does not exist" % workflow_definition_id)
  73. self._delete_workflow_associations(workflow_definition_id)
  74. self.clients.resource_registry.delete(workflow_definition_id)
  75. def create_data_process_workflow(self, workflow_definition_id='', input_data_product_id=''):
  76. """Instantiates a Data Process Workflow specified by a Workflow Definition resource and an input data product id.
  77. Returns the id of the workflow and the data product id for the final output product.
  78. @param workflow_definition_id str
  79. @param input_data_product_id str
  80. @retval workflow_id str
  81. @retval output_data_product_id str
  82. @throws BadRequest if any of the required parameters are not set
  83. @throws NotFound object with specified id does not exist
  84. """
  85. if not workflow_definition_id:
  86. raise BadRequest("The workflow_definition_id parameter is missing")
  87. workflow_definition = self.clients.resource_registry.read(workflow_definition_id)
  88. if not workflow_definition:
  89. raise NotFound("WorkflowDefinition %s does not exist" % workflow_definition_id)
  90. if not input_data_product_id:
  91. raise BadRequest("The input_data_product_id parameter is missing")
  92. input_data_product = self.clients.resource_registry.read(input_data_product_id)
  93. if not input_data_product:
  94. raise NotFound("The input data product %s does not exist" % input_data_product_id)
  95. #Create Workflow object and associations to track the instantiation of a work flow definition.
  96. workflow = IonObject(RT.Workflow, name=workflow_definition.name)
  97. workflow_id, _ = self.clients.resource_registry.create(workflow)
  98. self.clients.resource_registry.create_association(workflow_id, PRED.hasDefinition,workflow_definition_id )
  99. self.clients.resource_registry.create_association(workflow_id, PRED.hasInputProduct,input_data_product_id )
  100. #Setup the input data product id as the initial input product stream
  101. data_process_input_dp_id = input_data_product_id
  102. output_data_product_id = None
  103. #Iterate through the workflow steps to setup the data processes and connect them together.
  104. for wf_step in workflow_definition.workflow_steps:
  105. log.debug("wf_step.data_process_definition_id: " + wf_step.data_process_definition_id)
  106. data_process_definition = self.clients.resource_registry.read(wf_step.data_process_definition_id)
  107. # Find the link between the output Stream Definition resource and the Data Process Definition resource
  108. stream_ids,_ = self.clients.resource_registry.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True)
  109. if not stream_ids:
  110. raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id )
  111. process_output_stream_def_id = stream_ids[0]
  112. #If an output name has been specified than use it for the final output product name
  113. if wf_step.output_data_product_name is not '':
  114. data_product_name = wf_step.output_data_product_name
  115. else:
  116. #Concatenate the name of the workflow and data process definition for the name of the data product output + plus
  117. #a unique identifier for multiple instances of a workflow definition.
  118. data_product_name = workflow_definition.name + '_' + data_process_definition.name + '_' + workflow_id
  119. # Create the output data product of the transform
  120. transform_dp_obj = IonObject(RT.DataProduct, name=data_product_name,description=data_process_definition.description)
  121. transform_dp_id = self.clients.data_product_management.create_data_product(transform_dp_obj, process_output_stream_def_id)
  122. if wf_step.persist_process_output_data:
  123. self.clients.data_product_management.activate_data_product_persistence(data_product_id=transform_dp_id, persist_data=wf_step.persist_process_output_data, persist_metadata=wf_step.persist_process_output_data)
  124. #Associate the intermediate data products with the workflow
  125. self.clients.resource_registry.create_association(workflow_id, PRED.hasDataProduct, transform_dp_id )
  126. # Create the transform data process
  127. log.debug("create data_process and start it")
  128. data_process_id = self.clients.data_process_management.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id}, configuration=wf_step.configuration)
  129. self.clients.data_process_management.activate_data_process(data_process_id)
  130. #Track the the data process with an association to the workflow
  131. self.clients.resource_registry.create_association(workflow_id, PRED.hasDataProcess, data_process_id )
  132. #last one out of the for loop is the output product id
  133. output_data_product_id = transform_dp_id
  134. #Save the id of the output data stream for input to the next process in the workflow.
  135. data_process_input_dp_id = transform_dp_id
  136. #Track the output data product with an association
  137. self.clients.resource_registry.create_association(workflow_id, PRED.hasOutputProduct, output_data_product_id )
  138. return workflow_id, output_data_product_id
  139. def terminate_data_process_workflow(self, workflow_id='', delete_data_products=True):
  140. """Terminates a Workflow specific by a Workflow Definition resource which includes all internal processes.
  141. @param workflow_id str
  142. @throws BadRequest if any of the required parameters are not set
  143. @throws NotFound object with specified id does not exist
  144. """
  145. if not workflow_id:
  146. raise BadRequest("The workflow_id parameter is missing")
  147. workflow = self.clients.resource_registry.read(workflow_id)
  148. if not workflow:
  149. raise NotFound("Workflow %s does not exist" % workflow_id)
  150. #Iterate through all of the data process associates and deactivate and delete them
  151. process_ids,_ = self.clients.resource_registry.find_objects(workflow_id, PRED.hasDataProcess, RT.DataProcess, True)
  152. for pid in process_ids:
  153. self.clients.data_process_management.deactivate_data_process(pid)
  154. self.clients.data_process_management.delete_data_process(pid)
  155. aid = self.clients.resource_registry.find_associations(workflow_id, PRED.hasDataProcess, pid)
  156. if aid:
  157. self.clients.resource_registry.delete_association(aid[0])
  158. #Iterate through all of the data product associations and suspend and delete them
  159. workflow_dp_ids,_ = self.clients.resource_registry.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
  160. for dp_id in workflow_dp_ids:
  161. if delete_data_products: #TODO - may have to revisit this once the SA level stabilizes
  162. self.clients.data_product_management.suspend_data_product_persistence(dp_id)
  163. self.clients.data_product_management.delete_data_product(dp_id)
  164. aid = self.clients.resource_registry.find_associations(workflow_id, PRED.hasDataProduct, dp_id)
  165. if aid:
  166. self.clients.resource_registry.delete_association(aid[0])
  167. #Remove other associations
  168. aid = self.clients.resource_registry.find_associations(workflow_id, PRED.hasOutputProduct)
  169. if aid:
  170. self.clients.resource_registry.delete_association(aid[0])
  171. aid = self.clients.resource_registry.find_associations(workflow_id, PRED.hasInputProduct)
  172. if aid:
  173. self.clients.resource_registry.delete_association(aid[0])
  174. aid = self.clients.resource_registry.find_associations(workflow_id, PRED.hasDefinition)
  175. if aid:
  176. self.clients.resource_registry.delete_association(aid[0])
  177. #Finally remove the workflow object itself
  178. self.clients.resource_registry.delete(workflow_id)