/python_modules/dagster/dagster/core/definitions/preset.py

https://github.com/dagster-io/dagster · Python · 214 lines · 201 code · 5 blank · 8 comment · 0 complexity · 886e2c7b2ac7e2c2f724dcc279a1b55f MD5 · raw file

  1. from collections import namedtuple
  2. import pkg_resources
  3. import six
  4. import yaml
  5. from dagster import check
  6. from dagster.core.definitions.utils import config_from_files, config_from_yaml_strings
  7. from dagster.core.errors import DagsterInvariantViolationError
  8. from dagster.seven import FileNotFoundError, ModuleNotFoundError # pylint:disable=redefined-builtin
  9. from dagster.utils import merge_dicts
  10. from dagster.utils.backcompat import canonicalize_backcompat_args
  11. from .mode import DEFAULT_MODE_NAME
  12. class PresetDefinition(
  13. namedtuple("_PresetDefinition", "name run_config solid_selection mode tags")
  14. ):
  15. """Defines a preset configuration in which a pipeline can execute.
  16. Presets can be used in Dagit to load predefined configurations into the tool.
  17. Presets may also be used from the Python API (in a script, or in test) as follows:
  18. .. code-block:: python
  19. execute_pipeline(pipeline_def, preset='example_preset')
  20. Presets may also be used with the command line tools:
  21. .. code-block:: shell
  22. $ dagster pipeline execute example_pipeline --preset example_preset
  23. Args:
  24. name (str): The name of this preset. Must be unique in the presets defined on a given
  25. pipeline.
  26. run_config (Optional[dict]): A dict representing the config to set with the preset.
  27. This is equivalent to the ``run_config`` argument to :py:func:`execute_pipeline`.
  28. solid_selection (Optional[List[str]]): A list of solid subselection (including single
  29. solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']``
  30. mode (Optional[str]): The mode to apply when executing this preset. (default: 'default')
  31. tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
  32. """
  33. def __new__(
  34. cls, name, run_config=None, solid_selection=None, mode=None, tags=None,
  35. ):
  36. return super(PresetDefinition, cls).__new__(
  37. cls,
  38. name=check.str_param(name, "name"),
  39. run_config=run_config,
  40. solid_selection=check.opt_nullable_list_param(
  41. solid_selection, "solid_selection", of_type=str
  42. ),
  43. mode=check.opt_str_param(mode, "mode", DEFAULT_MODE_NAME),
  44. tags=check.opt_dict_param(tags, "tags", key_type=str),
  45. )
  46. @staticmethod
  47. def from_files(
  48. name, environment_files=None, config_files=None, solid_selection=None, mode=None, tags=None
  49. ):
  50. """Static constructor for presets from YAML files.
  51. Args:
  52. name (str): The name of this preset. Must be unique in the presets defined on a given
  53. pipeline.
  54. config_files (Optional[List[str]]): List of paths or glob patterns for yaml files
  55. to load and parse as the environment config for this preset.
  56. solid_selection (Optional[List[str]]): A list of solid subselection (including single
  57. solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']``
  58. mode (Optional[str]): The mode to apply when executing this preset. (default:
  59. 'default')
  60. tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
  61. Returns:
  62. PresetDefinition: A PresetDefinition constructed from the provided YAML files.
  63. Raises:
  64. DagsterInvariantViolationError: When one of the YAML files is invalid and has a parse
  65. error.
  66. """
  67. check.str_param(name, "name")
  68. config_files = canonicalize_backcompat_args(
  69. config_files, "config_files", environment_files, "environment_files", "0.9.0"
  70. )
  71. config_files = check.opt_list_param(config_files, "config_files")
  72. solid_selection = check.opt_nullable_list_param(
  73. solid_selection, "solid_selection", of_type=str
  74. )
  75. mode = check.opt_str_param(mode, "mode", DEFAULT_MODE_NAME)
  76. merged = config_from_files(config_files)
  77. return PresetDefinition(name, merged, solid_selection, mode, tags)
  78. @staticmethod
  79. def from_yaml_strings(name, yaml_strings=None, solid_selection=None, mode=None, tags=None):
  80. """Static constructor for presets from YAML strings.
  81. Args:
  82. name (str): The name of this preset. Must be unique in the presets defined on a given
  83. pipeline.
  84. yaml_strings (Optional[List[str]]): List of yaml strings to parse as the environment
  85. config for this preset.
  86. solid_selection (Optional[List[str]]): A list of solid subselection (including single
  87. solid names) to execute with the preset. e.g. ``['*some_solid+', 'other_solid']``
  88. mode (Optional[str]): The mode to apply when executing this preset. (default:
  89. 'default')
  90. tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
  91. Returns:
  92. PresetDefinition: A PresetDefinition constructed from the provided YAML strings
  93. Raises:
  94. DagsterInvariantViolationError: When one of the YAML documents is invalid and has a
  95. parse error.
  96. """
  97. check.str_param(name, "name")
  98. yaml_strings = check.opt_list_param(yaml_strings, "yaml_strings", of_type=str)
  99. solid_selection = check.opt_nullable_list_param(
  100. solid_selection, "solid_selection", of_type=str
  101. )
  102. mode = check.opt_str_param(mode, "mode", DEFAULT_MODE_NAME)
  103. merged = config_from_yaml_strings(yaml_strings)
  104. return PresetDefinition(name, merged, solid_selection, mode, tags)
  105. @staticmethod
  106. def from_pkg_resources(
  107. name, pkg_resource_defs=None, solid_selection=None, mode=None, tags=None
  108. ):
  109. """Load a preset from a package resource, using :py:func:`pkg_resources.resource_string`.
  110. Example:
  111. .. code-block:: python
  112. PresetDefinition.from_pkg_resources(
  113. name='local',
  114. mode='local',
  115. pkg_resource_defs=[
  116. ('dagster_examples.airline_demo.environments', 'local_base.yaml'),
  117. ('dagster_examples.airline_demo.environments', 'local_warehouse.yaml'),
  118. ],
  119. )
  120. Args:
  121. name (str): The name of this preset. Must be unique in the presets defined on a given
  122. pipeline.
  123. pkg_resource_defs (Optional[List[(str, str)]]): List of pkg_resource modules/files to
  124. load as environment config for this preset.
  125. solid_selection (Optional[List[str]]): A list of solid subselection (including single
  126. solid names) to execute with this partition. e.g.
  127. ``['*some_solid+', 'other_solid']``
  128. mode (Optional[str]): The mode to apply when executing this preset. (default:
  129. 'default')
  130. tags (Optional[Dict[str, Any]]): The tags to apply when executing this preset.
  131. Returns:
  132. PresetDefinition: A PresetDefinition constructed from the provided YAML strings
  133. Raises:
  134. DagsterInvariantViolationError: When one of the YAML documents is invalid and has a
  135. parse error.
  136. """
  137. pkg_resource_defs = check.opt_list_param(
  138. pkg_resource_defs, "pkg_resource_defs", of_type=tuple
  139. )
  140. try:
  141. yaml_strings = [
  142. six.ensure_str(pkg_resources.resource_string(*pkg_resource_def))
  143. for pkg_resource_def in pkg_resource_defs
  144. ]
  145. except (ModuleNotFoundError, FileNotFoundError, UnicodeDecodeError) as err:
  146. six.raise_from(
  147. DagsterInvariantViolationError(
  148. "Encountered error attempting to parse yaml. Loading YAMLs from "
  149. "package resources {pkg_resource_defs} "
  150. 'on preset "{name}".'.format(pkg_resource_defs=pkg_resource_defs, name=name)
  151. ),
  152. err,
  153. )
  154. return PresetDefinition.from_yaml_strings(name, yaml_strings, solid_selection, mode, tags)
  155. def get_environment_yaml(self):
  156. """Get the environment dict set on a preset as YAML.
  157. Returns:
  158. str: The environment dict as YAML.
  159. """
  160. return yaml.dump(self.run_config or {}, default_flow_style=False)
  161. def with_additional_config(self, run_config):
  162. """Return a new PresetDefinition with additional config merged in to the existing config."""
  163. check.opt_nullable_dict_param(run_config, "run_config")
  164. if run_config is None:
  165. return self
  166. else:
  167. return PresetDefinition(
  168. name=self.name,
  169. solid_selection=self.solid_selection,
  170. mode=self.mode,
  171. tags=self.tags,
  172. run_config=merge_dicts(self.run_config, run_config),
  173. )