/conans/client/loader.py

https://github.com/conan-io/conan · Python · 398 lines · 294 code · 60 blank · 44 comment · 102 complexity · 9139fae40c31ab1c6b1d163a5cf158f5 MD5 · raw file

  1. import fnmatch
  2. import imp
  3. import inspect
  4. import os
  5. import sys
  6. import uuid
  7. import yaml
  8. from conans.client.conf.required_version import validate_conan_version
  9. from conans.client.generators import registered_generators
  10. from conans.client.loader_txt import ConanFileTextLoader
  11. from conans.client.tools.files import chdir
  12. from conans.errors import ConanException, NotFoundException, ConanInvalidConfiguration, \
  13. conanfile_exception_formatter
  14. from conans.model.conan_file import ConanFile
  15. from conans.model.conan_generator import Generator
  16. from conans.model.options import OptionsValues
  17. from conans.model.ref import ConanFileReference
  18. from conans.model.settings import Settings
  19. from conans.model.values import Values
  20. from conans.paths import DATA_YML
  21. from conans.util.conan_v2_mode import CONAN_V2_MODE_ENVVAR
  22. from conans.util.files import load
  23. class ConanFileLoader(object):
  24. def __init__(self, runner, output, python_requires, pyreq_loader=None):
  25. self._runner = runner
  26. self._output = output
  27. self._pyreq_loader = pyreq_loader
  28. self._python_requires = python_requires
  29. sys.modules["conans"].python_requires = python_requires
  30. self._cached_conanfile_classes = {}
  31. def load_basic(self, conanfile_path, lock_python_requires=None, user=None, channel=None,
  32. display=""):
  33. """ loads a conanfile basic object without evaluating anything
  34. """
  35. return self.load_basic_module(conanfile_path, lock_python_requires, user, channel,
  36. display)[0]
  37. def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None, channel=None,
  38. display=""):
  39. """ loads a conanfile basic object without evaluating anything, returns the module too
  40. """
  41. cached = self._cached_conanfile_classes.get(conanfile_path)
  42. if cached and cached[1] == lock_python_requires:
  43. return cached[0](self._output, self._runner, display, user, channel), cached[2]
  44. if lock_python_requires is not None:
  45. self._python_requires.locked_versions = {r.name: r for r in lock_python_requires}
  46. try:
  47. self._python_requires.valid = True
  48. module, conanfile = parse_conanfile(conanfile_path, self._python_requires)
  49. self._python_requires.valid = False
  50. self._python_requires.locked_versions = None
  51. # This is the new py_requires feature, to supersede the old python_requires
  52. if self._pyreq_loader:
  53. self._pyreq_loader.load_py_requires(conanfile, lock_python_requires, self)
  54. conanfile.recipe_folder = os.path.dirname(conanfile_path)
  55. # If the scm is inherited, create my own instance
  56. if hasattr(conanfile, "scm") and "scm" not in conanfile.__class__.__dict__:
  57. if isinstance(conanfile.scm, dict):
  58. conanfile.scm = conanfile.scm.copy()
  59. # Load and populate dynamic fields from the data file
  60. conan_data = self._load_data(conanfile_path)
  61. conanfile.conan_data = conan_data
  62. if conan_data and '.conan' in conan_data:
  63. scm_data = conan_data['.conan'].get('scm')
  64. if scm_data:
  65. conanfile.scm.update(scm_data)
  66. self._cached_conanfile_classes[conanfile_path] = (conanfile, lock_python_requires,
  67. module)
  68. result = conanfile(self._output, self._runner, display, user, channel)
  69. if hasattr(result, "init") and callable(result.init):
  70. with conanfile_exception_formatter(str(result), "init"):
  71. result.init()
  72. return result, module
  73. except ConanException as e:
  74. raise ConanException("Error loading conanfile at '{}': {}".format(conanfile_path, e))
  75. @staticmethod
  76. def _load_data(conanfile_path):
  77. data_path = os.path.join(os.path.dirname(conanfile_path), DATA_YML)
  78. if not os.path.exists(data_path):
  79. return None
  80. try:
  81. data = yaml.safe_load(load(data_path))
  82. except Exception as e:
  83. raise ConanException("Invalid yml format at {}: {}".format(DATA_YML, e))
  84. return data or {}
  85. def load_named(self, conanfile_path, name, version, user, channel, lock_python_requires=None):
  86. """ loads the basic conanfile object and evaluates its name and version
  87. """
  88. conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires, user, channel)
  89. # Export does a check on existing name & version
  90. if name:
  91. if conanfile.name and name != conanfile.name:
  92. raise ConanException("Package recipe with name %s!=%s" % (name, conanfile.name))
  93. conanfile.name = name
  94. if version:
  95. if conanfile.version and version != conanfile.version:
  96. raise ConanException("Package recipe with version %s!=%s"
  97. % (version, conanfile.version))
  98. conanfile.version = version
  99. if hasattr(conanfile, "set_name"):
  100. with conanfile_exception_formatter("conanfile.py", "set_name"):
  101. conanfile.set_name()
  102. if name and name != conanfile.name:
  103. raise ConanException("Package recipe with name %s!=%s" % (name, conanfile.name))
  104. if hasattr(conanfile, "set_version"):
  105. with conanfile_exception_formatter("conanfile.py", "set_version"):
  106. conanfile.set_version()
  107. if version and version != conanfile.version:
  108. raise ConanException("Package recipe with version %s!=%s"
  109. % (version, conanfile.version))
  110. return conanfile
  111. def load_export(self, conanfile_path, name, version, user, channel, lock_python_requires=None):
  112. """ loads the conanfile and evaluates its name, version, and enforce its existence
  113. """
  114. conanfile = self.load_named(conanfile_path, name, version, user, channel,
  115. lock_python_requires)
  116. if not conanfile.name:
  117. raise ConanException("conanfile didn't specify name")
  118. if not conanfile.version:
  119. raise ConanException("conanfile didn't specify version")
  120. if os.environ.get(CONAN_V2_MODE_ENVVAR, False):
  121. conanfile.version = str(conanfile.version)
  122. ref = ConanFileReference(conanfile.name, conanfile.version, user, channel)
  123. conanfile.display_name = str(ref)
  124. conanfile.output.scope = conanfile.display_name
  125. return conanfile
  126. @staticmethod
  127. def _initialize_conanfile(conanfile, profile):
  128. # Prepare the settings for the loaded conanfile
  129. # Mixing the global settings with the specified for that name if exist
  130. tmp_settings = profile.processed_settings.copy()
  131. package_settings_values = profile.package_settings_values
  132. if package_settings_values:
  133. pkg_settings = package_settings_values.get(conanfile.name)
  134. if pkg_settings is None:
  135. # FIXME: This seems broken for packages without user/channel
  136. ref = "%s/%s@%s/%s" % (conanfile.name, conanfile.version,
  137. conanfile._conan_user, conanfile._conan_channel)
  138. for pattern, settings in package_settings_values.items():
  139. if fnmatch.fnmatchcase(ref, pattern):
  140. pkg_settings = settings
  141. break
  142. if pkg_settings:
  143. tmp_settings.values = Values.from_list(pkg_settings)
  144. conanfile.initialize(tmp_settings, profile.env_values)
  145. def load_consumer(self, conanfile_path, profile_host, name=None, version=None, user=None,
  146. channel=None, lock_python_requires=None):
  147. """ loads a conanfile.py in user space. Might have name/version or not
  148. """
  149. conanfile = self.load_named(conanfile_path, name, version, user, channel,
  150. lock_python_requires)
  151. ref = ConanFileReference(conanfile.name, conanfile.version, user, channel, validate=False)
  152. if str(ref):
  153. conanfile.display_name = "%s (%s)" % (os.path.basename(conanfile_path), str(ref))
  154. else:
  155. conanfile.display_name = os.path.basename(conanfile_path)
  156. conanfile.output.scope = conanfile.display_name
  157. conanfile.in_local_cache = False
  158. try:
  159. self._initialize_conanfile(conanfile, profile_host)
  160. # The consumer specific
  161. conanfile.develop = True
  162. profile_host.user_options.descope_options(conanfile.name)
  163. conanfile.options.initialize_upstream(profile_host.user_options,
  164. name=conanfile.name)
  165. profile_host.user_options.clear_unscoped_options()
  166. return conanfile
  167. except ConanInvalidConfiguration:
  168. raise
  169. except Exception as e: # re-raise with file name
  170. raise ConanException("%s: %s" % (conanfile_path, str(e)))
  171. def load_conanfile(self, conanfile_path, profile, ref, lock_python_requires=None):
  172. """ load a conanfile with a full reference, name, version, user and channel are obtained
  173. from the reference, not evaluated. Main way to load from the cache
  174. """
  175. try:
  176. conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires,
  177. ref.user, ref.channel, str(ref))
  178. except Exception as e:
  179. raise ConanException("%s: Cannot load recipe.\n%s" % (str(ref), str(e)))
  180. conanfile.name = ref.name
  181. conanfile.version = str(ref.version) \
  182. if os.environ.get(CONAN_V2_MODE_ENVVAR, False) else ref.version
  183. if profile.dev_reference and profile.dev_reference == ref:
  184. conanfile.develop = True
  185. try:
  186. self._initialize_conanfile(conanfile, profile)
  187. return conanfile
  188. except ConanInvalidConfiguration:
  189. raise
  190. except Exception as e: # re-raise with file name
  191. raise ConanException("%s: %s" % (conanfile_path, str(e)))
  192. def load_conanfile_txt(self, conan_txt_path, profile_host, ref=None):
  193. if not os.path.exists(conan_txt_path):
  194. raise NotFoundException("Conanfile not found!")
  195. contents = load(conan_txt_path)
  196. path, basename = os.path.split(conan_txt_path)
  197. display_name = "%s (%s)" % (basename, ref) if ref and ref.name else basename
  198. conanfile = self._parse_conan_txt(contents, path, display_name, profile_host)
  199. return conanfile
  200. def _parse_conan_txt(self, contents, path, display_name, profile):
  201. conanfile = ConanFile(self._output, self._runner, display_name)
  202. conanfile.initialize(Settings(), profile.env_values)
  203. # It is necessary to copy the settings, because the above is only a constraint of
  204. # conanfile settings, and a txt doesn't define settings. Necessary for generators,
  205. # as cmake_multi, that check build_type.
  206. conanfile.settings = profile.processed_settings.copy_values()
  207. try:
  208. parser = ConanFileTextLoader(contents)
  209. except Exception as e:
  210. raise ConanException("%s:\n%s" % (path, str(e)))
  211. for reference in parser.requirements:
  212. ref = ConanFileReference.loads(reference) # Raise if invalid
  213. conanfile.requires.add_ref(ref)
  214. for build_reference in parser.build_requirements:
  215. ConanFileReference.loads(build_reference)
  216. if not hasattr(conanfile, "build_requires"):
  217. conanfile.build_requires = []
  218. conanfile.build_requires.append(build_reference)
  219. conanfile.generators = parser.generators
  220. try:
  221. options = OptionsValues.loads(parser.options)
  222. except Exception:
  223. raise ConanException("Error while parsing [options] in conanfile\n"
  224. "Options should be specified as 'pkg:option=value'")
  225. conanfile.options.values = options
  226. conanfile.options.initialize_upstream(profile.user_options)
  227. # imports method
  228. conanfile.imports = parser.imports_method(conanfile)
  229. return conanfile
  230. def load_virtual(self, references, profile_host, scope_options=True,
  231. build_requires_options=None):
  232. # If user don't specify namespace in options, assume that it is
  233. # for the reference (keep compatibility)
  234. conanfile = ConanFile(self._output, self._runner, display_name="virtual")
  235. conanfile.initialize(profile_host.processed_settings.copy(),
  236. profile_host.env_values)
  237. conanfile.settings = profile_host.processed_settings.copy_values()
  238. for reference in references:
  239. conanfile.requires.add_ref(reference)
  240. # Allows options without package namespace in conan install commands:
  241. # conan install zlib/1.2.8@lasote/stable -o shared=True
  242. if scope_options:
  243. assert len(references) == 1
  244. profile_host.user_options.scope_options(references[0].name)
  245. if build_requires_options:
  246. conanfile.options.initialize_upstream(build_requires_options)
  247. else:
  248. conanfile.options.initialize_upstream(profile_host.user_options)
  249. conanfile.generators = [] # remove the default txt generator
  250. return conanfile
  251. def _parse_module(conanfile_module, module_id):
  252. """ Parses a python in-memory module, to extract the classes, mainly the main
  253. class defining the Recipe, but also process possible existing generators
  254. @param conanfile_module: the module to be processed
  255. @return: the main ConanFile class from the module
  256. """
  257. result = None
  258. for name, attr in conanfile_module.__dict__.items():
  259. if (name.startswith("_") or not inspect.isclass(attr) or
  260. attr.__dict__.get("__module__") != module_id):
  261. continue
  262. if issubclass(attr, ConanFile) and attr != ConanFile:
  263. if result is None:
  264. result = attr
  265. else:
  266. raise ConanException("More than 1 conanfile in the file")
  267. elif issubclass(attr, Generator) and attr != Generator:
  268. registered_generators.add(attr.__name__, attr, custom=True)
  269. if result is None:
  270. raise ConanException("No subclass of ConanFile")
  271. return result
  272. def parse_conanfile(conanfile_path, python_requires):
  273. with python_requires.capture_requires() as py_requires:
  274. module, filename = _parse_conanfile(conanfile_path)
  275. try:
  276. conanfile = _parse_module(module, filename)
  277. # Check for duplicates
  278. # TODO: move it into PythonRequires
  279. py_reqs = {}
  280. for it in py_requires:
  281. if it.ref.name in py_reqs:
  282. dupes = [str(it.ref), str(py_reqs[it.ref.name].ref)]
  283. raise ConanException("Same python_requires with different versions not allowed"
  284. " for a conanfile. Found '{}'".format("', '".join(dupes)))
  285. py_reqs[it.ref.name] = it
  286. # Make them available to the conanfile itself
  287. if py_reqs:
  288. conanfile.python_requires = py_reqs
  289. return module, conanfile
  290. except Exception as e: # re-raise with file name
  291. raise ConanException("%s: %s" % (conanfile_path, str(e)))
  292. def _parse_conanfile(conan_file_path):
  293. """ From a given path, obtain the in memory python import module
  294. """
  295. if not os.path.exists(conan_file_path):
  296. raise NotFoundException("%s not found!" % conan_file_path)
  297. module_id = str(uuid.uuid1())
  298. current_dir = os.path.dirname(conan_file_path)
  299. sys.path.insert(0, current_dir)
  300. try:
  301. old_modules = list(sys.modules.keys())
  302. with chdir(current_dir):
  303. sys.dont_write_bytecode = True
  304. loaded = imp.load_source(module_id, conan_file_path)
  305. sys.dont_write_bytecode = False
  306. required_conan_version = getattr(loaded, "required_conan_version", None)
  307. if required_conan_version:
  308. validate_conan_version(required_conan_version)
  309. # These lines are necessary, otherwise local conanfile imports with same name
  310. # collide, but no error, and overwrite other packages imports!!
  311. added_modules = set(sys.modules).difference(old_modules)
  312. for added in added_modules:
  313. module = sys.modules[added]
  314. if module:
  315. try:
  316. try:
  317. # Most modules will have __file__ != None
  318. folder = os.path.dirname(module.__file__)
  319. except (AttributeError, TypeError):
  320. # But __file__ might not exist or equal None
  321. # Like some builtins and Namespace packages py3
  322. folder = module.__path__._path[0]
  323. except AttributeError: # In case the module.__path__ doesn't exist
  324. pass
  325. else:
  326. if folder.startswith(current_dir):
  327. module = sys.modules.pop(added)
  328. sys.modules["%s.%s" % (module_id, added)] = module
  329. except ConanException:
  330. raise
  331. except Exception:
  332. import traceback
  333. trace = traceback.format_exc().split('\n')
  334. raise ConanException("Unable to load conanfile in %s\n%s" % (conan_file_path,
  335. '\n'.join(trace[3:])))
  336. finally:
  337. sys.path.pop(0)
  338. return loaded, module_id