/python_modules/dagster/dagster/core/execution/context/system.py

https://github.com/dagster-io/dagster · Python · 359 lines · 261 code · 71 blank · 27 comment · 9 complexity · 12390b08af5e5a7cd5ae4795c678429d MD5 · raw file

  1. """
  2. This module contains the execution context objects that are internal to the system.
  3. Not every property on these should be exposed to random Jane or Joe dagster user
  4. so we have a different layer of objects that encode the explicit public API
  5. in the user_context module
  6. """
  7. from collections import namedtuple
  8. from dagster import check
  9. from dagster.core.definitions.hook import HookDefinition
  10. from dagster.core.definitions.mode import ModeDefinition
  11. from dagster.core.definitions.pipeline_base import IPipeline
  12. from dagster.core.definitions.resource import ScopedResourcesBuilder
  13. from dagster.core.definitions.step_launcher import StepLauncher
  14. from dagster.core.errors import DagsterInvariantViolationError
  15. from dagster.core.execution.retries import Retries
  16. from dagster.core.executor.base import Executor
  17. from dagster.core.log_manager import DagsterLogManager
  18. from dagster.core.storage.file_manager import FileManager
  19. from dagster.core.storage.pipeline_run import PipelineRun
  20. from dagster.core.system_config.objects import EnvironmentConfig
  21. class SystemExecutionContextData(
  22. namedtuple(
  23. "_SystemExecutionContextData",
  24. (
  25. "pipeline_run scoped_resources_builder environment_config pipeline "
  26. "mode_def system_storage_def intermediate_storage_def instance intermediate_storage file_manager "
  27. "raise_on_error retries"
  28. ),
  29. )
  30. ):
  31. """
  32. SystemExecutionContextData is the data that remains constant throughout the entire
  33. execution of a pipeline or plan.
  34. """
  35. def __new__(
  36. cls,
  37. pipeline_run,
  38. scoped_resources_builder,
  39. environment_config,
  40. pipeline,
  41. mode_def,
  42. system_storage_def,
  43. intermediate_storage_def,
  44. instance,
  45. intermediate_storage,
  46. file_manager,
  47. raise_on_error,
  48. retries,
  49. ):
  50. from dagster.core.definitions.system_storage import SystemStorageDefinition
  51. from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition
  52. from dagster.core.storage.intermediate_storage import IntermediateStorage
  53. from dagster.core.instance import DagsterInstance
  54. return super(SystemExecutionContextData, cls).__new__(
  55. cls,
  56. pipeline_run=check.inst_param(pipeline_run, "pipeline_run", PipelineRun),
  57. scoped_resources_builder=check.inst_param(
  58. scoped_resources_builder, "scoped_resources_builder", ScopedResourcesBuilder
  59. ),
  60. environment_config=check.inst_param(
  61. environment_config, "environment_config", EnvironmentConfig
  62. ),
  63. pipeline=check.inst_param(pipeline, "pipeline", IPipeline),
  64. mode_def=check.inst_param(mode_def, "mode_def", ModeDefinition),
  65. system_storage_def=check.inst_param(
  66. system_storage_def, "system_storage_def", SystemStorageDefinition
  67. ),
  68. intermediate_storage_def=check.opt_inst_param(
  69. intermediate_storage_def, "intermediate_storage_def", IntermediateStorageDefinition
  70. ),
  71. instance=check.inst_param(instance, "instance", DagsterInstance),
  72. intermediate_storage=check.inst_param(
  73. intermediate_storage, "intermediate_storage", IntermediateStorage
  74. ),
  75. file_manager=check.inst_param(file_manager, "file_manager", FileManager),
  76. raise_on_error=check.bool_param(raise_on_error, "raise_on_error"),
  77. retries=check.inst_param(retries, "retries", Retries),
  78. )
  79. @property
  80. def run_id(self):
  81. return self.pipeline_run.run_id
  82. @property
  83. def run_config(self):
  84. return self.environment_config.original_config_dict
  85. @property
  86. def pipeline_def(self):
  87. return self.pipeline.get_definition()
  88. class SystemExecutionContext(object):
  89. __slots__ = ["_execution_context_data", "_log_manager"]
  90. def __init__(self, execution_context_data, log_manager):
  91. self._execution_context_data = check.inst_param(
  92. execution_context_data, "execution_context_data", SystemExecutionContextData
  93. )
  94. self._log_manager = check.inst_param(log_manager, "log_manager", DagsterLogManager)
  95. @property
  96. def pipeline_run(self):
  97. return self._execution_context_data.pipeline_run
  98. @property
  99. def scoped_resources_builder(self):
  100. return self._execution_context_data.scoped_resources_builder
  101. @property
  102. def run_id(self):
  103. return self._execution_context_data.run_id
  104. @property
  105. def run_config(self):
  106. return self._execution_context_data.run_config
  107. @property
  108. def environment_config(self):
  109. return self._execution_context_data.environment_config
  110. @property
  111. def pipeline(self):
  112. return self._execution_context_data.pipeline
  113. @property
  114. def pipeline_def(self):
  115. return self._execution_context_data.pipeline_def
  116. @property
  117. def mode_def(self):
  118. return self._execution_context_data.mode_def
  119. @property
  120. def system_storage_def(self):
  121. return self._execution_context_data.system_storage_def
  122. @property
  123. def intermediate_storage_def(self):
  124. return self._execution_context_data.intermediate_storage_def
  125. @property
  126. def instance(self):
  127. return self._execution_context_data.instance
  128. @property
  129. def intermediate_storage(self):
  130. return self._execution_context_data.intermediate_storage
  131. @property
  132. def file_manager(self):
  133. return self._execution_context_data.file_manager
  134. @property
  135. def raise_on_error(self):
  136. return self._execution_context_data.raise_on_error
  137. @property
  138. def retries(self):
  139. return self._execution_context_data.retries
  140. @property
  141. def log(self):
  142. return self._log_manager
  143. @property
  144. def logging_tags(self):
  145. return self._log_manager.logging_tags
  146. def has_tag(self, key):
  147. check.str_param(key, "key")
  148. return key in self.logging_tags
  149. def get_tag(self, key):
  150. check.str_param(key, "key")
  151. return self.logging_tags.get(key)
  152. def for_step(self, step):
  153. from dagster.core.execution.plan.objects import ExecutionStep
  154. check.inst_param(step, "step", ExecutionStep)
  155. return SystemStepExecutionContext(
  156. self._execution_context_data, self._log_manager.with_tags(**step.logging_tags), step,
  157. )
  158. def for_type(self, dagster_type):
  159. return TypeCheckContext(self._execution_context_data, self.log, dagster_type)
  160. class SystemPipelineExecutionContext(SystemExecutionContext):
  161. __slots__ = ["_executor"]
  162. def __init__(self, execution_context_data, log_manager, executor):
  163. super(SystemPipelineExecutionContext, self).__init__(execution_context_data, log_manager)
  164. self._executor = check.inst_param(executor, "executor", Executor)
  165. @property
  166. def executor(self):
  167. return self._executor
  168. class SystemStepExecutionContext(SystemExecutionContext):
  169. __slots__ = ["_step", "_resources", "_required_resource_keys", "_step_launcher"]
  170. def __init__(self, execution_context_data, log_manager, step):
  171. from dagster.core.execution.plan.objects import ExecutionStep
  172. from dagster.core.execution.resources_init import get_required_resource_keys_for_step
  173. self._step = check.inst_param(step, "step", ExecutionStep)
  174. super(SystemStepExecutionContext, self).__init__(execution_context_data, log_manager)
  175. self._required_resource_keys = get_required_resource_keys_for_step(
  176. step,
  177. execution_context_data.pipeline_def,
  178. execution_context_data.system_storage_def,
  179. execution_context_data.intermediate_storage_def,
  180. )
  181. self._resources = self._execution_context_data.scoped_resources_builder.build(
  182. self._required_resource_keys
  183. )
  184. step_launcher_resources = [
  185. resource for resource in self._resources if isinstance(resource, StepLauncher)
  186. ]
  187. if len(step_launcher_resources) > 1:
  188. raise DagsterInvariantViolationError(
  189. "Multiple required resources for solid {solid_name} have inherit StepLauncher"
  190. "There should be at most one step launcher resource per solid.".format(
  191. solid_name=step.solid_handle.name
  192. )
  193. )
  194. elif len(step_launcher_resources) == 1:
  195. self._step_launcher = step_launcher_resources[0]
  196. else:
  197. self._step_launcher = None
  198. self._log_manager = log_manager
  199. def for_compute(self):
  200. return SystemComputeExecutionContext(self._execution_context_data, self.log, self.step)
  201. @property
  202. def step(self):
  203. return self._step
  204. @property
  205. def step_launcher(self):
  206. return self._step_launcher
  207. @property
  208. def solid_handle(self):
  209. return self._step.solid_handle
  210. @property
  211. def solid_def(self):
  212. return self.solid.definition
  213. @property
  214. def solid(self):
  215. return self.pipeline_def.get_solid(self._step.solid_handle)
  216. @property
  217. def resources(self):
  218. return self._resources
  219. @property
  220. def required_resource_keys(self):
  221. return self._required_resource_keys
  222. @property
  223. def log(self):
  224. return self._log_manager
  225. def for_hook(self, hook_def):
  226. return HookContext(self._execution_context_data, self.log, hook_def, self.step)
  227. class SystemComputeExecutionContext(SystemStepExecutionContext):
  228. @property
  229. def solid_config(self):
  230. solid_config = self.environment_config.solids.get(str(self.solid_handle))
  231. return solid_config.config if solid_config else None
  232. class TypeCheckContext(SystemExecutionContext):
  233. """The ``context`` object available to a type check function on a DagsterType.
  234. Attributes:
  235. log (DagsterLogManager): Centralized log dispatch from user code.
  236. resources (Any): An object whose attributes contain the resources available to this solid.
  237. run_id (str): The id of this pipeline run.
  238. """
  239. def __init__(self, execution_context_data, log_manager, dagster_type):
  240. super(TypeCheckContext, self).__init__(execution_context_data, log_manager)
  241. self._resources = self._execution_context_data.scoped_resources_builder.build(
  242. dagster_type.required_resource_keys
  243. )
  244. self._log_manager = log_manager
  245. @property
  246. def resources(self):
  247. return self._resources
  248. class HookContext(SystemExecutionContext):
  249. """The ``context`` object available to a hook function on an DagsterEvent.
  250. Attributes:
  251. log (DagsterLogManager): Centralized log dispatch from user code.
  252. hook_def (HookDefinition): The hook that the context object belongs to.
  253. step (ExecutionStep): The compute step associated with the hook.
  254. solid (Solid): The solid instance associated with the hook.
  255. resources (Any): Resources available in the hook context.
  256. solid_config (Dict[str, Any]): The parsed config specific to this solid.
  257. """
  258. def __init__(self, execution_context_data, log_manager, hook_def, step):
  259. from dagster.core.execution.plan.objects import ExecutionStep
  260. super(HookContext, self).__init__(execution_context_data, log_manager)
  261. self._log_manager = log_manager
  262. self._hook_def = check.inst_param(hook_def, "hook_def", HookDefinition)
  263. self._step = check.inst_param(step, "step", ExecutionStep)
  264. self._required_resource_keys = hook_def.required_resource_keys
  265. self._resources = self._execution_context_data.scoped_resources_builder.build(
  266. self._required_resource_keys
  267. )
  268. @property
  269. def hook_def(self):
  270. return self._hook_def
  271. @property
  272. def step(self):
  273. return self._step
  274. @property
  275. def solid(self):
  276. return self.pipeline_def.get_solid(self._step.solid_handle)
  277. @property
  278. def resources(self):
  279. return self._resources
  280. @property
  281. def required_resource_keys(self):
  282. return self._required_resource_keys
  283. @property
  284. def solid_config(self):
  285. solid_config = self.environment_config.solids.get(str(self._step.solid_handle))
  286. return solid_config.config if solid_config else None