/impl/cuddlefish/packaging.py

https://github.com/CoderPuppy/chromeless · Python · 349 lines · 326 code · 14 blank · 9 comment · 0 complexity · d0eb0f6b480d3b7aa2ac5c59c31b5691 MD5 · raw file

  1. import os
  2. import sys
  3. import re
  4. import copy
  5. import simplejson as json
  6. from cuddlefish.bunch import Bunch
  7. from manifest import scan_package, update_manifest_with_fileinfo
  8. MANIFEST_NAME = 'package.json'
  9. DEFAULT_LOADER = 'api-utils'
  10. DEFAULT_PROGRAM_MODULE = 'main'
  11. DEFAULT_ICON = 'icon.png'
  12. DEFAULT_ICON64 = 'icon64.png'
  13. METADATA_PROPS = ['name', 'description', 'keywords', 'author', 'version',
  14. 'contributors', 'license', 'url', 'icon', 'icon64']
  15. RESOURCE_HOSTNAME_RE = re.compile(r'^[a-z0-9_\-]+$')
  16. class Error(Exception):
  17. pass
  18. class MalformedPackageError(Error):
  19. pass
  20. class MalformedJsonFileError(Error):
  21. pass
  22. class DuplicatePackageError(Error):
  23. pass
  24. class PackageNotFoundError(Error):
  25. def __init__(self, missing_package, reason):
  26. self.missing_package = missing_package
  27. self.reason = reason
  28. def __str__(self):
  29. return "%s (%s)" % (self.missing_package, self.reason)
  30. class BadChromeMarkerError(Error):
  31. pass
  32. def validate_resource_hostname(name):
  33. """
  34. Validates the given hostname for a resource: URI.
  35. For more information, see:
  36. https://bugzilla.mozilla.org/show_bug.cgi?id=566812#c13
  37. Examples:
  38. >>> validate_resource_hostname('blarg')
  39. >>> validate_resource_hostname('BLARG')
  40. Traceback (most recent call last):
  41. ...
  42. ValueError: invalid resource hostname: BLARG
  43. >>> validate_resource_hostname('foo@bar')
  44. Traceback (most recent call last):
  45. ...
  46. ValueError: invalid resource hostname: foo@bar
  47. """
  48. if not RESOURCE_HOSTNAME_RE.match(name):
  49. raise ValueError('invalid resource hostname: %s' % name)
  50. def find_packages_with_module(pkg_cfg, name):
  51. # TODO: Make this support more than just top-level modules.
  52. filename = "%s.js" % name
  53. packages = []
  54. for cfg in pkg_cfg.packages.itervalues():
  55. if 'lib' in cfg:
  56. matches = [dirname for dirname in resolve_dirs(cfg, cfg.lib)
  57. if os.path.exists(os.path.join(dirname, filename))]
  58. if matches:
  59. packages.append(cfg.name)
  60. return packages
  61. def resolve_dirs(pkg_cfg, dirnames):
  62. for dirname in dirnames:
  63. yield resolve_dir(pkg_cfg, dirname)
  64. def resolve_dir(pkg_cfg, dirname):
  65. return os.path.join(pkg_cfg.root_dir, dirname)
  66. def get_metadata(pkg_cfg, deps):
  67. metadata = Bunch()
  68. for pkg_name in deps:
  69. cfg = pkg_cfg.packages[pkg_name]
  70. metadata[pkg_name] = Bunch()
  71. for prop in METADATA_PROPS:
  72. if cfg.get(prop):
  73. metadata[pkg_name][prop] = cfg[prop]
  74. return metadata
  75. def apply_default_dir(base_json, base_path, dirname):
  76. if (not base_json.get(dirname) and
  77. os.path.isdir(os.path.join(base_path, dirname))):
  78. base_json[dirname] = dirname
  79. def normalize_string_or_array(base_json, key):
  80. if base_json.get(key):
  81. if isinstance(base_json[key], basestring):
  82. base_json[key] = [base_json[key]]
  83. def load_json_file(path):
  84. data = open(path, 'r').read()
  85. try:
  86. return Bunch(json.loads(data))
  87. except ValueError, e:
  88. raise MalformedJsonFileError('%s when reading "%s"' % (str(e),
  89. path))
  90. def get_config_in_dir(path):
  91. package_json = os.path.join(path, MANIFEST_NAME)
  92. if not (os.path.exists(package_json) and
  93. os.path.isfile(package_json)):
  94. raise MalformedPackageError('%s not found in "%s"' % (MANIFEST_NAME,
  95. path))
  96. base_json = load_json_file(package_json)
  97. if 'name' not in base_json:
  98. base_json.name = os.path.basename(path)
  99. for dirname in ['lib', 'tests', 'data', 'packages']:
  100. apply_default_dir(base_json, path, dirname)
  101. if (not base_json.get('icon') and
  102. os.path.isfile(os.path.join(path, DEFAULT_ICON))):
  103. base_json['icon'] = DEFAULT_ICON
  104. if (not base_json.get('icon64') and
  105. os.path.isfile(os.path.join(path, DEFAULT_ICON64))):
  106. base_json['icon64'] = DEFAULT_ICON64
  107. for key in ['lib', 'tests', 'dependencies', 'packages']:
  108. normalize_string_or_array(base_json, key)
  109. if 'main' not in base_json and 'lib' in base_json:
  110. for dirname in base_json['lib']:
  111. program = os.path.join(path, dirname,
  112. '%s.js' % DEFAULT_PROGRAM_MODULE)
  113. if os.path.exists(program):
  114. base_json['main'] = DEFAULT_PROGRAM_MODULE
  115. break
  116. base_json.root_dir = path
  117. return base_json
  118. def _is_same_file(a, b):
  119. if hasattr(os.path, 'samefile'):
  120. return os.path.samefile(a, b)
  121. return a == b
  122. def build_config(root_dir, target_cfg):
  123. dirs_to_scan = []
  124. def add_packages_from_config(pkgconfig):
  125. if 'packages' in pkgconfig:
  126. for package_dir in resolve_dirs(pkgconfig, pkgconfig.packages):
  127. dirs_to_scan.append(package_dir)
  128. add_packages_from_config(target_cfg)
  129. packages_dir = os.path.join(root_dir, 'packages')
  130. if os.path.exists(packages_dir) and os.path.isdir(packages_dir):
  131. dirs_to_scan.append(packages_dir)
  132. packages = Bunch({target_cfg.name: target_cfg})
  133. while dirs_to_scan:
  134. packages_dir = dirs_to_scan.pop()
  135. package_paths = [os.path.join(packages_dir, dirname)
  136. for dirname in os.listdir(packages_dir)
  137. if not dirname.startswith('.')]
  138. for path in package_paths:
  139. pkgconfig = get_config_in_dir(path)
  140. if pkgconfig.name in packages:
  141. otherpkg = packages[pkgconfig.name]
  142. if not _is_same_file(otherpkg.root_dir, path):
  143. raise DuplicatePackageError(path, otherpkg.root_dir)
  144. else:
  145. packages[pkgconfig.name] = pkgconfig
  146. add_packages_from_config(pkgconfig)
  147. return Bunch(packages=packages)
  148. def get_deps_for_targets(pkg_cfg, targets):
  149. visited = []
  150. deps_left = [[dep, None] for dep in list(targets)]
  151. while deps_left:
  152. [dep, required_by] = deps_left.pop()
  153. if dep not in visited:
  154. visited.append(dep)
  155. if dep not in pkg_cfg.packages:
  156. required_reason = ("required by '%s'" % (required_by)) \
  157. if required_by is not None \
  158. else "specified as target"
  159. raise PackageNotFoundError(dep, required_reason)
  160. dep_cfg = pkg_cfg.packages[dep]
  161. deps_left.extend([[i, dep] for i in dep_cfg.get('dependencies', [])])
  162. return visited
  163. def generate_build_for_target(pkg_cfg, target, deps, prefix='',
  164. include_tests=True,
  165. include_dep_tests=False,
  166. default_loader=DEFAULT_LOADER):
  167. validate_resource_hostname(prefix)
  168. manifest = {}
  169. build = Bunch(resources=Bunch(),
  170. resourcePackages=Bunch(),
  171. packageData=Bunch(),
  172. rootPaths=[],
  173. manifest=manifest,
  174. )
  175. def add_section_to_build(cfg, section, is_code=False,
  176. is_data=False):
  177. if section in cfg:
  178. dirnames = cfg[section]
  179. if isinstance(dirnames, basestring):
  180. # This is just for internal consistency within this
  181. # function, it has nothing to do w/ a non-canonical
  182. # configuration dict.
  183. dirnames = [dirnames]
  184. for dirname in resolve_dirs(cfg, dirnames):
  185. name = "-".join([prefix + cfg.name,
  186. os.path.basename(dirname)])
  187. validate_resource_hostname(name)
  188. if name in build.resources:
  189. raise KeyError('resource already defined', name)
  190. build.resourcePackages[name] = cfg.name
  191. build.resources[name] = dirname
  192. resource_url = 'resource://%s/' % name
  193. if is_code:
  194. build.rootPaths.insert(0, resource_url)
  195. pkg_manifest, problems = scan_package(prefix, resource_url,
  196. cfg.name,
  197. section, dirname)
  198. if problems:
  199. # the relevant instructions have already been written
  200. # to stderr
  201. raise BadChromeMarkerError()
  202. manifest.update(pkg_manifest)
  203. if is_data:
  204. build.packageData[cfg.name] = resource_url
  205. def add_dep_to_build(dep):
  206. dep_cfg = pkg_cfg.packages[dep]
  207. add_section_to_build(dep_cfg, "lib", is_code=True)
  208. add_section_to_build(dep_cfg, "data", is_data=True)
  209. if include_tests and include_dep_tests:
  210. add_section_to_build(dep_cfg, "tests", is_code=True)
  211. if ("loader" in dep_cfg) and ("loader" not in build):
  212. build.loader = "resource://%s-%s" % (prefix + dep,
  213. dep_cfg.loader)
  214. target_cfg = pkg_cfg.packages[target]
  215. if include_tests and not include_dep_tests:
  216. add_section_to_build(target_cfg, "tests", is_code=True)
  217. for dep in deps:
  218. add_dep_to_build(dep)
  219. if 'loader' not in build:
  220. add_dep_to_build(DEFAULT_LOADER)
  221. if 'icon' in target_cfg:
  222. build['icon'] = os.path.join(target_cfg.root_dir, target_cfg.icon)
  223. del target_cfg['icon']
  224. if 'icon64' in target_cfg:
  225. build['icon64'] = os.path.join(target_cfg.root_dir, target_cfg.icon64)
  226. del target_cfg['icon64']
  227. # now go back through and find out where each module lives, to record the
  228. # pathname in the manifest
  229. update_manifest_with_fileinfo(deps, DEFAULT_LOADER, manifest)
  230. return build
  231. def _get_files_in_dir(path):
  232. data = {}
  233. files = os.listdir(path)
  234. for filename in files:
  235. fullpath = os.path.join(path, filename)
  236. if os.path.isdir(fullpath):
  237. data[filename] = _get_files_in_dir(fullpath)
  238. else:
  239. try:
  240. info = os.stat(fullpath)
  241. data[filename] = dict(size=info.st_size)
  242. except OSError:
  243. pass
  244. return data
  245. def build_pkg_index(pkg_cfg):
  246. pkg_cfg = copy.deepcopy(pkg_cfg)
  247. for pkg in pkg_cfg.packages:
  248. root_dir = pkg_cfg.packages[pkg].root_dir
  249. files = _get_files_in_dir(root_dir)
  250. pkg_cfg.packages[pkg].files = files
  251. try:
  252. readme = open(root_dir + '/README.md').read()
  253. pkg_cfg.packages[pkg].readme = readme
  254. except IOError:
  255. pass
  256. del pkg_cfg.packages[pkg].root_dir
  257. return pkg_cfg.packages
  258. def build_pkg_cfg(root):
  259. pkg_cfg = build_config(root, Bunch(name='dummy'))
  260. del pkg_cfg.packages['dummy']
  261. return pkg_cfg
  262. def call_plugins(pkg_cfg, deps):
  263. for dep in deps:
  264. dep_cfg = pkg_cfg.packages[dep]
  265. dirnames = dep_cfg.get('python-lib', [])
  266. for dirname in resolve_dirs(dep_cfg, dirnames):
  267. sys.path.append(dirname)
  268. module_names = dep_cfg.get('python-plugins', [])
  269. for module_name in module_names:
  270. module = __import__(module_name)
  271. module.init(root_dir=dep_cfg.root_dir)
  272. def call_cmdline_tool(env_root, pkg_name):
  273. pkg_cfg = build_config(env_root, Bunch(name='dummy'))
  274. if pkg_name not in pkg_cfg.packages:
  275. print "This tool requires the '%s' package." % pkg_name
  276. sys.exit(1)
  277. cfg = pkg_cfg.packages[pkg_name]
  278. for dirname in resolve_dirs(cfg, cfg['python-lib']):
  279. sys.path.append(dirname)
  280. module_name = cfg.get('python-cmdline-tool')
  281. module = __import__(module_name)
  282. module.run()