/kapitan/inputs/base.py

https://github.com/deepmind/kapitan
Python | 169 lines | 118 code | 28 blank | 23 comment | 18 complexity | f5a798f6a9c8297495e2400885b02472 MD5 | raw file
  1. #!/usr/bin/env python3
  2. # Copyright 2019 The Kapitan Authors
  3. # SPDX-FileCopyrightText: 2020 The Kapitan Authors <kapitan-admins@googlegroups.com>
  4. #
  5. # SPDX-License-Identifier: Apache-2.0
  6. import glob
  7. import itertools
  8. import json
  9. import logging
  10. import os
  11. from collections.abc import Mapping
  12. import yaml
  13. from kapitan.errors import CompileError, KapitanError
  14. from kapitan.refs.base import Revealer
  15. from kapitan.utils import PrettyDumper
  16. logger = logging.getLogger(__name__)
  17. class InputType(object):
  18. def __init__(self, type_name, compile_path, search_paths, ref_controller, inventory_path="inventory/"):
  19. self.type_name = type_name
  20. self.compile_path = compile_path
  21. self.search_paths = search_paths
  22. self.ref_controller = ref_controller
  23. self.inventory_path = inventory_path
  24. def compile_obj(self, comp_obj, ext_vars, **kwargs):
  25. """
  26. Expand globbed input paths, taking into account provided search paths
  27. and run compile_input_path() for each resolved input_path.
  28. kwargs are passed into compile_input_path()
  29. """
  30. input_type = comp_obj["input_type"]
  31. assert input_type == self.type_name
  32. # expand any globbed paths, taking into account provided search paths
  33. input_paths = []
  34. for input_path in comp_obj["input_paths"]:
  35. globbed_paths = [glob.glob(os.path.join(path, input_path)) for path in self.search_paths]
  36. inputs = list(itertools.chain.from_iterable(globbed_paths))
  37. # remove duplicate inputs
  38. inputs = set(inputs)
  39. if len(inputs) == 0:
  40. raise CompileError(
  41. "Compile error: {} for target: {} not found in "
  42. "search_paths: {}".format(input_path, ext_vars["target"], self.search_paths)
  43. )
  44. input_paths.extend(inputs)
  45. for input_path in input_paths:
  46. self.compile_input_path(input_path, comp_obj, ext_vars, **kwargs)
  47. def compile_input_path(self, input_path, comp_obj, ext_vars, **kwargs):
  48. """
  49. Compile validated input_path in comp_obj
  50. kwargs are passed into compile_file()
  51. """
  52. target_name = ext_vars["target"]
  53. output_path = comp_obj["output_path"]
  54. output_type = comp_obj.get("output_type", self.default_output_type())
  55. prune_input = comp_obj.get("prune", kwargs.get("prune", False))
  56. logger.debug("Compiling %s", input_path)
  57. try:
  58. _compile_path = os.path.join(self.compile_path, target_name, output_path)
  59. self.compile_file(
  60. input_path,
  61. _compile_path,
  62. ext_vars,
  63. output=output_type,
  64. target_name=target_name,
  65. prune_input=prune_input,
  66. **kwargs,
  67. )
  68. except KapitanError as e:
  69. raise CompileError("{}\nCompile error: failed to compile target: {}".format(e, target_name))
  70. def make_compile_dirs(self, target_name, output_path):
  71. """make compile dirs, skips if dirs exist"""
  72. _compile_path = os.path.join(self.compile_path, target_name, output_path)
  73. # support writing to an already existent dir
  74. os.makedirs(_compile_path, exist_ok=True)
  75. def compile_file(self, file_path, compile_path, ext_vars, **kwargs):
  76. """implements compilation for file_path to compile_path with ext_vars"""
  77. return NotImplementedError
  78. def default_output_type(self):
  79. "returns default output_type value"
  80. return NotImplementedError
  81. class CompilingFile(object):
  82. def __init__(self, context, fp, ref_controller, **kwargs):
  83. self.fp = fp
  84. self.ref_controller = ref_controller
  85. self.kwargs = kwargs
  86. self.revealer = Revealer(ref_controller)
  87. def write(self, data):
  88. """write data into file"""
  89. reveal = self.kwargs.get("reveal", False)
  90. target_name = self.kwargs.get("target_name", None)
  91. if reveal:
  92. self.fp.write(self.revealer.reveal_raw(data))
  93. else:
  94. self.fp.write(self.revealer.compile_raw(data, target_name=target_name))
  95. def write_yaml(self, obj):
  96. """recursively compile or reveal refs and convert obj to yaml and write to file"""
  97. indent = self.kwargs.get("indent", 2)
  98. reveal = self.kwargs.get("reveal", False)
  99. target_name = self.kwargs.get("target_name", None)
  100. if reveal:
  101. obj = self.revealer.reveal_obj(obj)
  102. else:
  103. obj = self.revealer.compile_obj(obj, target_name=target_name)
  104. if obj:
  105. if isinstance(obj, Mapping):
  106. yaml.dump(obj, stream=self.fp, indent=indent, Dumper=PrettyDumper, default_flow_style=False)
  107. else:
  108. yaml.dump_all(
  109. obj, stream=self.fp, indent=indent, Dumper=PrettyDumper, default_flow_style=False
  110. )
  111. logger.debug("Wrote %s", self.fp.name)
  112. else:
  113. logger.debug("%s is Empty, skipped writing output", self.fp.name)
  114. def write_json(self, obj):
  115. """recursively hash or reveal refs and convert obj to json and write to file"""
  116. indent = self.kwargs.get("indent", 2)
  117. reveal = self.kwargs.get("reveal", False)
  118. target_name = self.kwargs.get("target_name", None)
  119. if reveal:
  120. obj = self.revealer.reveal_obj(obj)
  121. else:
  122. obj = self.revealer.compile_obj(obj, target_name=target_name)
  123. if obj:
  124. json.dump(obj, self.fp, indent=indent)
  125. logger.debug("Wrote %s", self.fp.name)
  126. else:
  127. logger.debug("%s is Empty, skipped writing output", self.fp.name)
  128. class CompiledFile(object):
  129. def __init__(self, name, ref_controller, **kwargs):
  130. self.name = name
  131. self.fp = None
  132. self.ref_controller = ref_controller
  133. self.kwargs = kwargs
  134. def __enter__(self):
  135. mode = self.kwargs.get("mode", "r")
  136. # make sure directory for file exists
  137. os.makedirs(os.path.dirname(self.name), exist_ok=True)
  138. self.fp = open(self.name, mode)
  139. return CompilingFile(self, self.fp, self.ref_controller, **self.kwargs)
  140. def __exit__(self, *args):
  141. self.fp.close()