PageRenderTime 51ms CodeModel.GetById 16ms RepoModel.GetById 0ms app.codeStats 0ms

/fdroidserver/metadata.py

https://gitlab.com/smichel17/fdroidserver
Python | 988 lines | 897 code | 48 blank | 43 comment | 91 complexity | 011d9e149a5f4e57166ec7fdc053dce3 MD5 | raw file
  1. #!/usr/bin/env python3
  2. #
  3. # metadata.py - part of the FDroid server tools
  4. # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
  5. # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
  6. # Copyright (C) 2017-2018 Michael Pöhn <michael.poehn@fsfe.org>
  7. #
  8. # This program is free software: you can redistribute it and/or modify
  9. # it under the terms of the GNU Affero General Public License as published by
  10. # the Free Software Foundation, either version 3 of the License, or
  11. # (at your option) any later version.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU Affero General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU Affero General Public License
  19. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  20. import os
  21. import re
  22. import glob
  23. import logging
  24. import yaml
  25. try:
  26. from yaml import CSafeLoader as SafeLoader
  27. except ImportError:
  28. from yaml import SafeLoader
  29. import importlib
  30. from collections import OrderedDict
  31. import fdroidserver.common
  32. from fdroidserver import _
  33. from fdroidserver.exception import MetaDataException, FDroidException
  34. srclibs = None
  35. warnings_action = None
  36. # validates usernames based on a loose collection of rules from GitHub, GitLab,
  37. # Liberapay and issuehunt. This is mostly to block abuse.
  38. VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECASE)
  39. def _warn_or_exception(value, cause=None):
  40. '''output warning or Exception depending on -W'''
  41. if warnings_action == 'ignore':
  42. pass
  43. elif warnings_action == 'error':
  44. if cause:
  45. raise MetaDataException(value) from cause
  46. else:
  47. raise MetaDataException(value)
  48. else:
  49. logging.warning(value)
  50. yaml_app_field_order = [
  51. 'Disabled',
  52. 'AntiFeatures',
  53. 'Categories',
  54. 'License',
  55. 'AuthorName',
  56. 'AuthorEmail',
  57. 'AuthorWebSite',
  58. 'WebSite',
  59. 'SourceCode',
  60. 'IssueTracker',
  61. 'Translation',
  62. 'Changelog',
  63. 'Donate',
  64. 'FlattrID',
  65. 'Liberapay',
  66. 'LiberapayID',
  67. 'OpenCollective',
  68. 'Bitcoin',
  69. 'Litecoin',
  70. '\n',
  71. 'Name',
  72. 'AutoName',
  73. 'Summary',
  74. 'Description',
  75. '\n',
  76. 'RequiresRoot',
  77. '\n',
  78. 'RepoType',
  79. 'Repo',
  80. 'Binaries',
  81. '\n',
  82. 'Builds',
  83. '\n',
  84. 'MaintainerNotes',
  85. '\n',
  86. 'ArchivePolicy',
  87. 'AutoUpdateMode',
  88. 'UpdateCheckMode',
  89. 'UpdateCheckIgnore',
  90. 'VercodeOperation',
  91. 'UpdateCheckName',
  92. 'UpdateCheckData',
  93. 'CurrentVersion',
  94. 'CurrentVersionCode',
  95. '\n',
  96. 'NoSourceSince',
  97. ]
  98. yaml_app_fields = [x for x in yaml_app_field_order if x != '\n']
  99. class App(dict):
  100. def __init__(self, copydict=None):
  101. if copydict:
  102. super().__init__(copydict)
  103. return
  104. super().__init__()
  105. self.Disabled = None
  106. self.AntiFeatures = []
  107. self.Provides = None
  108. self.Categories = []
  109. self.License = 'Unknown'
  110. self.AuthorName = None
  111. self.AuthorEmail = None
  112. self.AuthorWebSite = None
  113. self.WebSite = ''
  114. self.SourceCode = ''
  115. self.IssueTracker = ''
  116. self.Translation = ''
  117. self.Changelog = ''
  118. self.Donate = None
  119. self.FlattrID = None
  120. self.Liberapay = None
  121. self.LiberapayID = None
  122. self.OpenCollective = None
  123. self.Bitcoin = None
  124. self.Litecoin = None
  125. self.Name = None
  126. self.AutoName = ''
  127. self.Summary = ''
  128. self.Description = ''
  129. self.RequiresRoot = False
  130. self.RepoType = ''
  131. self.Repo = ''
  132. self.Binaries = None
  133. self.MaintainerNotes = ''
  134. self.ArchivePolicy = None
  135. self.AutoUpdateMode = 'None'
  136. self.UpdateCheckMode = 'None'
  137. self.UpdateCheckIgnore = None
  138. self.VercodeOperation = None
  139. self.UpdateCheckName = None
  140. self.UpdateCheckData = None
  141. self.CurrentVersion = ''
  142. self.CurrentVersionCode = None
  143. self.NoSourceSince = ''
  144. self.id = None
  145. self.metadatapath = None
  146. self.Builds = []
  147. self.comments = {}
  148. self.added = None
  149. self.lastUpdated = None
  150. def __getattr__(self, name):
  151. if name in self:
  152. return self[name]
  153. else:
  154. raise AttributeError("No such attribute: " + name)
  155. def __setattr__(self, name, value):
  156. self[name] = value
  157. def __delattr__(self, name):
  158. if name in self:
  159. del self[name]
  160. else:
  161. raise AttributeError("No such attribute: " + name)
  162. def get_last_build(self):
  163. if len(self.Builds) > 0:
  164. return self.Builds[-1]
  165. else:
  166. return Build()
  167. TYPE_UNKNOWN = 0
  168. TYPE_OBSOLETE = 1
  169. TYPE_STRING = 2
  170. TYPE_BOOL = 3
  171. TYPE_LIST = 4
  172. TYPE_SCRIPT = 5
  173. TYPE_MULTILINE = 6
  174. TYPE_BUILD = 7
  175. TYPE_INT = 8
  176. fieldtypes = {
  177. 'Description': TYPE_MULTILINE,
  178. 'MaintainerNotes': TYPE_MULTILINE,
  179. 'Categories': TYPE_LIST,
  180. 'AntiFeatures': TYPE_LIST,
  181. 'Build': TYPE_BUILD,
  182. 'BuildVersion': TYPE_OBSOLETE,
  183. 'UseBuilt': TYPE_OBSOLETE,
  184. }
  185. def fieldtype(name):
  186. name = name.replace(' ', '')
  187. if name in fieldtypes:
  188. return fieldtypes[name]
  189. return TYPE_STRING
  190. # In the order in which they are laid out on files
  191. build_flags = [
  192. 'versionName',
  193. 'versionCode',
  194. 'disable',
  195. 'commit',
  196. 'timeout',
  197. 'subdir',
  198. 'submodules',
  199. 'sudo',
  200. 'init',
  201. 'patch',
  202. 'gradle',
  203. 'maven',
  204. 'buildozer',
  205. 'output',
  206. 'srclibs',
  207. 'oldsdkloc',
  208. 'encoding',
  209. 'forceversion',
  210. 'forcevercode',
  211. 'rm',
  212. 'extlibs',
  213. 'prebuild',
  214. 'androidupdate',
  215. 'target',
  216. 'scanignore',
  217. 'scandelete',
  218. 'build',
  219. 'buildjni',
  220. 'ndk',
  221. 'preassemble',
  222. 'gradleprops',
  223. 'antcommands',
  224. 'novcheck',
  225. 'antifeatures',
  226. ]
  227. class Build(dict):
  228. def __init__(self, copydict=None):
  229. super().__init__()
  230. self.disable = ''
  231. self.commit = None
  232. self.timeout = None
  233. self.subdir = None
  234. self.submodules = False
  235. self.sudo = ''
  236. self.init = ''
  237. self.patch = []
  238. self.gradle = []
  239. self.maven = False
  240. self.buildozer = False
  241. self.output = None
  242. self.srclibs = []
  243. self.oldsdkloc = False
  244. self.encoding = None
  245. self.forceversion = False
  246. self.forcevercode = False
  247. self.rm = []
  248. self.extlibs = []
  249. self.prebuild = ''
  250. self.androidupdate = []
  251. self.target = None
  252. self.scanignore = []
  253. self.scandelete = []
  254. self.build = ''
  255. self.buildjni = []
  256. self.ndk = None
  257. self.preassemble = []
  258. self.gradleprops = []
  259. self.antcommands = []
  260. self.novcheck = False
  261. self.antifeatures = []
  262. if copydict:
  263. super().__init__(copydict)
  264. return
  265. def __getattr__(self, name):
  266. if name in self:
  267. return self[name]
  268. else:
  269. raise AttributeError("No such attribute: " + name)
  270. def __setattr__(self, name, value):
  271. self[name] = value
  272. def __delattr__(self, name):
  273. if name in self:
  274. del self[name]
  275. else:
  276. raise AttributeError("No such attribute: " + name)
  277. def build_method(self):
  278. for f in ['maven', 'gradle', 'buildozer']:
  279. if self.get(f):
  280. return f
  281. if self.output:
  282. return 'raw'
  283. return 'ant'
  284. # like build_method, but prioritize output=
  285. def output_method(self):
  286. if self.output:
  287. return 'raw'
  288. for f in ['maven', 'gradle', 'buildozer']:
  289. if self.get(f):
  290. return f
  291. return 'ant'
  292. def ndk_path(self):
  293. version = self.ndk
  294. if not version:
  295. version = 'r12b' # falls back to latest
  296. paths = fdroidserver.common.config['ndk_paths']
  297. if version not in paths:
  298. return ''
  299. return paths[version]
  300. flagtypes = {
  301. 'versionCode': TYPE_INT,
  302. 'extlibs': TYPE_LIST,
  303. 'srclibs': TYPE_LIST,
  304. 'patch': TYPE_LIST,
  305. 'rm': TYPE_LIST,
  306. 'buildjni': TYPE_LIST,
  307. 'preassemble': TYPE_LIST,
  308. 'androidupdate': TYPE_LIST,
  309. 'scanignore': TYPE_LIST,
  310. 'scandelete': TYPE_LIST,
  311. 'gradle': TYPE_LIST,
  312. 'antcommands': TYPE_LIST,
  313. 'gradleprops': TYPE_LIST,
  314. 'sudo': TYPE_SCRIPT,
  315. 'init': TYPE_SCRIPT,
  316. 'prebuild': TYPE_SCRIPT,
  317. 'build': TYPE_SCRIPT,
  318. 'submodules': TYPE_BOOL,
  319. 'oldsdkloc': TYPE_BOOL,
  320. 'forceversion': TYPE_BOOL,
  321. 'forcevercode': TYPE_BOOL,
  322. 'novcheck': TYPE_BOOL,
  323. 'antifeatures': TYPE_LIST,
  324. 'timeout': TYPE_INT,
  325. }
  326. def flagtype(name):
  327. if name in flagtypes:
  328. return flagtypes[name]
  329. return TYPE_STRING
  330. class FieldValidator():
  331. """
  332. Designates App metadata field types and checks that it matches
  333. 'name' - The long name of the field type
  334. 'matching' - List of possible values or regex expression
  335. 'sep' - Separator to use if value may be a list
  336. 'fields' - Metadata fields (Field:Value) of this type
  337. """
  338. def __init__(self, name, matching, fields):
  339. self.name = name
  340. self.matching = matching
  341. self.compiled = re.compile(matching)
  342. self.fields = fields
  343. def check(self, v, appid):
  344. if not v:
  345. return
  346. if type(v) == list:
  347. values = v
  348. else:
  349. values = [v]
  350. for v in values:
  351. if not self.compiled.match(v):
  352. _warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
  353. .format(value=v, field=self.name, appid=appid, pattern=self.matching))
  354. # Generic value types
  355. valuetypes = {
  356. FieldValidator("Flattr ID",
  357. r'^[0-9a-z]+$',
  358. ['FlattrID']),
  359. FieldValidator("Liberapay",
  360. VALID_USERNAME_REGEX,
  361. ['Liberapay']),
  362. FieldValidator("Liberapay ID",
  363. r'^[0-9]+$',
  364. ['LiberapayID']),
  365. FieldValidator("Open Collective",
  366. VALID_USERNAME_REGEX,
  367. ['OpenCollective']),
  368. FieldValidator("HTTP link",
  369. r'^http[s]?://',
  370. ["WebSite", "SourceCode", "IssueTracker", "Translation", "Changelog", "Donate"]),
  371. FieldValidator("Email",
  372. r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
  373. ["AuthorEmail"]),
  374. FieldValidator("Bitcoin address",
  375. r'^(bc1|[13])[a-zA-HJ-NP-Z0-9]{25,39}$',
  376. ["Bitcoin"]),
  377. FieldValidator("Litecoin address",
  378. r'^[LM3][a-km-zA-HJ-NP-Z1-9]{26,33}$',
  379. ["Litecoin"]),
  380. FieldValidator("Repo Type",
  381. r'^(git|git-svn|svn|hg|bzr|srclib)$',
  382. ["RepoType"]),
  383. FieldValidator("Binaries",
  384. r'^http[s]?://',
  385. ["Binaries"]),
  386. FieldValidator("Archive Policy",
  387. r'^[0-9]+ versions$',
  388. ["ArchivePolicy"]),
  389. FieldValidator("Anti-Feature",
  390. r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable|NoSourceSince)$',
  391. ["AntiFeatures"]),
  392. FieldValidator("Auto Update Mode",
  393. r"^(Version .+|None)$",
  394. ["AutoUpdateMode"]),
  395. FieldValidator("Update Check Mode",
  396. r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
  397. ["UpdateCheckMode"])
  398. }
  399. # Check an app's metadata information for integrity errors
  400. def check_metadata(app):
  401. for v in valuetypes:
  402. for k in v.fields:
  403. v.check(app[k], app.id)
  404. def parse_yaml_srclib(metadatapath):
  405. thisinfo = {'RepoType': '',
  406. 'Repo': '',
  407. 'Subdir': None,
  408. 'Prepare': None}
  409. if not os.path.exists(metadatapath):
  410. _warn_or_exception(_("Invalid scrlib metadata: '{file}' "
  411. "does not exist"
  412. .format(file=metadatapath)))
  413. return thisinfo
  414. with open(metadatapath, "r", encoding="utf-8") as f:
  415. try:
  416. data = yaml.load(f, Loader=SafeLoader)
  417. if type(data) is not dict:
  418. raise yaml.error.YAMLError(_('{file} is blank or corrupt!')
  419. .format(file=metadatapath))
  420. except yaml.error.YAMLError as e:
  421. _warn_or_exception(_("Invalid srclib metadata: could not "
  422. "parse '{file}'")
  423. .format(file=metadatapath) + '\n'
  424. + fdroidserver.common.run_yamllint(metadatapath,
  425. indent=4),
  426. cause=e)
  427. return thisinfo
  428. for key in data.keys():
  429. if key not in thisinfo.keys():
  430. _warn_or_exception(_("Invalid srclib metadata: unknown key "
  431. "'{key}' in '{file}'")
  432. .format(key=key, file=metadatapath))
  433. return thisinfo
  434. else:
  435. if key == 'Subdir':
  436. if isinstance(data[key], str):
  437. thisinfo[key] = data[key].split(',')
  438. elif isinstance(data[key], list):
  439. thisinfo[key] = data[key]
  440. elif data[key] is None:
  441. thisinfo[key] = ['']
  442. elif key == 'Prepare' and isinstance(data[key], list):
  443. thisinfo[key] = ' && '.join(data[key])
  444. else:
  445. thisinfo[key] = str(data[key] or '')
  446. return thisinfo
  447. def read_srclibs():
  448. """Read all srclib metadata.
  449. The information read will be accessible as metadata.srclibs, which is a
  450. dictionary, keyed on srclib name, with the values each being a dictionary
  451. in the same format as that returned by the parse_yaml_srclib function.
  452. A MetaDataException is raised if there are any problems with the srclib
  453. metadata.
  454. """
  455. global srclibs
  456. # They were already loaded
  457. if srclibs is not None:
  458. return
  459. srclibs = {}
  460. srcdir = 'srclibs'
  461. if not os.path.exists(srcdir):
  462. os.makedirs(srcdir)
  463. for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
  464. srclibname = os.path.basename(metadatapath[:-4])
  465. srclibs[srclibname] = parse_yaml_srclib(metadatapath)
  466. def read_metadata(appids={}, refresh=True, sort_by_time=False):
  467. """Return a list of App instances sorted newest first
  468. This reads all of the metadata files in a 'data' repository, then
  469. builds a list of App instances from those files. The list is
  470. sorted based on creation time, newest first. Most of the time,
  471. the newer files are the most interesting.
  472. appids is a dict with appids a keys and versionCodes as values.
  473. """
  474. # Always read the srclibs before the apps, since they can use a srlib as
  475. # their source repository.
  476. read_srclibs()
  477. apps = OrderedDict()
  478. for basedir in ('metadata', 'tmp'):
  479. if not os.path.exists(basedir):
  480. os.makedirs(basedir)
  481. if appids:
  482. vercodes = fdroidserver.common.read_pkg_args(appids)
  483. found_invalid = False
  484. metadatafiles = []
  485. for appid in vercodes.keys():
  486. f = os.path.join('metadata', '%s.yml' % appid)
  487. if os.path.exists(f):
  488. metadatafiles.append(f)
  489. else:
  490. found_invalid = True
  491. logging.critical(_("No such package: %s") % appid)
  492. if found_invalid:
  493. raise FDroidException(_("Found invalid appids in arguments"))
  494. else:
  495. metadatafiles = (glob.glob(os.path.join('metadata', '*.yml'))
  496. + glob.glob('.fdroid.yml'))
  497. if sort_by_time:
  498. entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
  499. metadatafiles = []
  500. for _ignored, path in sorted(entries, reverse=True):
  501. metadatafiles.append(path)
  502. else:
  503. # most things want the index alpha sorted for stability
  504. metadatafiles = sorted(metadatafiles)
  505. for metadatapath in metadatafiles:
  506. appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
  507. if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid):
  508. _warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!")
  509. .format(appid=appid, path=metadatapath))
  510. if appid in apps:
  511. _warn_or_exception(_("Found multiple metadata files for {appid}")
  512. .format(appid=appid))
  513. app = parse_metadata(metadatapath, appid in appids, refresh)
  514. check_metadata(app)
  515. apps[app.id] = app
  516. return apps
  517. # Port legacy ';' separators
  518. list_sep = re.compile(r'[,;]')
  519. def split_list_values(s):
  520. res = []
  521. for v in re.split(list_sep, s):
  522. if not v:
  523. continue
  524. v = v.strip()
  525. if not v:
  526. continue
  527. res.append(v)
  528. return res
  529. def sorted_builds(builds):
  530. return sorted(builds, key=lambda build: int(build.versionCode))
  531. esc_newlines = re.compile(r'\\( |\n)')
  532. def post_metadata_parse(app):
  533. # TODO keep native types, convert only for .txt metadata
  534. for k, v in app.items():
  535. if type(v) in (float, int):
  536. app[k] = str(v)
  537. if 'flavours' in app and app['flavours'] == [True]:
  538. app['flavours'] = 'yes'
  539. for field, fieldtype in fieldtypes.items():
  540. if fieldtype != TYPE_LIST:
  541. continue
  542. value = app.get(field)
  543. if isinstance(value, str):
  544. app[field] = [value, ]
  545. elif value is not None:
  546. app[field] = [str(i) for i in value]
  547. def _yaml_bool_unmapable(v):
  548. return v in (True, False, [True], [False])
  549. def _yaml_bool_unmap(v):
  550. if v is True:
  551. return 'yes'
  552. elif v is False:
  553. return 'no'
  554. elif v == [True]:
  555. return ['yes']
  556. elif v == [False]:
  557. return ['no']
  558. _bool_allowed = ('maven', 'buildozer')
  559. builds = []
  560. if 'Builds' in app:
  561. for build in app.get('Builds', []):
  562. if not isinstance(build, Build):
  563. build = Build(build)
  564. for k, v in build.items():
  565. if not (v is None):
  566. if flagtype(k) == TYPE_LIST:
  567. if _yaml_bool_unmapable(v):
  568. build[k] = _yaml_bool_unmap(v)
  569. if isinstance(v, str):
  570. build[k] = [v]
  571. elif isinstance(v, bool):
  572. if v:
  573. build[k] = ['yes']
  574. else:
  575. build[k] = []
  576. elif flagtype(k) is TYPE_INT:
  577. build[k] = str(v)
  578. elif flagtype(k) is TYPE_STRING:
  579. if isinstance(v, bool) and k in _bool_allowed:
  580. build[k] = v
  581. else:
  582. if _yaml_bool_unmapable(v):
  583. build[k] = _yaml_bool_unmap(v)
  584. else:
  585. build[k] = str(v)
  586. builds.append(build)
  587. app['Builds'] = sorted_builds(builds)
  588. # Parse metadata for a single application.
  589. #
  590. # 'metadatapath' - the filename to read. The "Application ID" aka
  591. # "Package Name" for the application comes from this
  592. # filename. Pass None to get a blank entry.
  593. #
  594. # Returns a dictionary containing all the details of the application. There are
  595. # two major kinds of information in the dictionary. Keys beginning with capital
  596. # letters correspond directory to identically named keys in the metadata file.
  597. # Keys beginning with lower case letters are generated in one way or another,
  598. # and are not found verbatim in the metadata.
  599. #
  600. # Known keys not originating from the metadata are:
  601. #
  602. # 'comments' - a list of comments from the metadata file. Each is
  603. # a list of the form [field, comment] where field is
  604. # the name of the field it preceded in the metadata
  605. # file. Where field is None, the comment goes at the
  606. # end of the file. Alternatively, 'build:version' is
  607. # for a comment before a particular build version.
  608. # 'descriptionlines' - original lines of description as formatted in the
  609. # metadata file.
  610. #
  611. bool_true = re.compile(r'([Yy]es|[Tt]rue)')
  612. bool_false = re.compile(r'([Nn]o|[Ff]alse)')
  613. def _decode_bool(s):
  614. if bool_true.match(s):
  615. return True
  616. if bool_false.match(s):
  617. return False
  618. _warn_or_exception(_("Invalid boolean '%s'") % s)
  619. def parse_metadata(metadatapath, check_vcs=False, refresh=True):
  620. '''parse metadata file, optionally checking the git repo for metadata first'''
  621. app = App()
  622. app.metadatapath = metadatapath
  623. name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
  624. if name == '.fdroid':
  625. check_vcs = False
  626. else:
  627. app.id = name
  628. if metadatapath.endswith('.yml'):
  629. with open(metadatapath, 'r') as mf:
  630. parse_yaml_metadata(mf, app)
  631. else:
  632. _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)')
  633. .format(path=metadatapath))
  634. if check_vcs and app.Repo:
  635. build_dir = fdroidserver.common.get_build_dir(app)
  636. metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
  637. if not os.path.isfile(metadata_in_repo):
  638. vcs, build_dir = fdroidserver.common.setup_vcs(app)
  639. if isinstance(vcs, fdroidserver.common.vcs_git):
  640. vcs.gotorevision('HEAD', refresh) # HEAD since we can't know where else to go
  641. if os.path.isfile(metadata_in_repo):
  642. logging.debug('Including metadata from ' + metadata_in_repo)
  643. # do not include fields already provided by main metadata file
  644. app_in_repo = parse_metadata(metadata_in_repo)
  645. for k, v in app_in_repo.items():
  646. if k not in app:
  647. app[k] = v
  648. post_metadata_parse(app)
  649. if not app.id:
  650. if app.get('Builds'):
  651. build = app['Builds'][-1]
  652. if build.subdir:
  653. root_dir = build.subdir
  654. else:
  655. root_dir = '.'
  656. paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
  657. _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
  658. return app
  659. def parse_yaml_metadata(mf, app):
  660. try:
  661. yamldata = yaml.load(mf, Loader=SafeLoader)
  662. except yaml.YAMLError as e:
  663. _warn_or_exception(_("could not parse '{path}'")
  664. .format(path=mf.name) + '\n'
  665. + fdroidserver.common.run_yamllint(mf.name,
  666. indent=4),
  667. cause=e)
  668. deprecated_in_yaml = ['Provides']
  669. if yamldata:
  670. for field in yamldata:
  671. if field not in yaml_app_fields:
  672. if field not in deprecated_in_yaml:
  673. _warn_or_exception(_("Unrecognised app field "
  674. "'{fieldname}' in '{path}'")
  675. .format(fieldname=field,
  676. path=mf.name))
  677. for deprecated_field in deprecated_in_yaml:
  678. if deprecated_field in yamldata:
  679. logging.warning(_("Ignoring '{field}' in '{metapath}' "
  680. "metadata because it is deprecated.")
  681. .format(field=deprecated_field,
  682. metapath=mf.name))
  683. del(yamldata[deprecated_field])
  684. if yamldata.get('Builds', None):
  685. for build in yamldata.get('Builds', []):
  686. # put all build flag keywords into a set to avoid
  687. # excessive looping action
  688. build_flag_set = set()
  689. for build_flag in build.keys():
  690. build_flag_set.add(build_flag)
  691. for build_flag in build_flag_set:
  692. if build_flag not in build_flags:
  693. _warn_or_exception(
  694. _("Unrecognised build flag '{build_flag}' "
  695. "in '{path}'").format(build_flag=build_flag,
  696. path=mf.name))
  697. post_parse_yaml_metadata(yamldata)
  698. app.update(yamldata)
  699. return app
  700. def post_parse_yaml_metadata(yamldata):
  701. """transform yaml metadata to our internal data format"""
  702. for build in yamldata.get('Builds', []):
  703. for flag in build.keys():
  704. _flagtype = flagtype(flag)
  705. if _flagtype is TYPE_SCRIPT:
  706. # concatenate script flags into a single string if they are stored as list
  707. if isinstance(build[flag], list):
  708. build[flag] = ' && '.join(build[flag])
  709. elif _flagtype is TYPE_STRING:
  710. # things like versionNames are strings, but without quotes can be numbers
  711. if isinstance(build[flag], float) or isinstance(build[flag], int):
  712. build[flag] = str(build[flag])
  713. elif _flagtype is TYPE_INT:
  714. # versionCode must be int
  715. if not isinstance(build[flag], int):
  716. _warn_or_exception(_('{build_flag} must be an integer, found: {value}')
  717. .format(build_flag=flag, value=build[flag]))
  718. def write_yaml(mf, app):
  719. """Write metadata in yaml format.
  720. :param mf: active file discriptor for writing
  721. :param app: app metadata to written to the yaml file
  722. """
  723. # import rumael.yaml and check version
  724. try:
  725. import ruamel.yaml
  726. except ImportError as e:
  727. raise FDroidException('ruamel.yaml not installed, can not write metadata.') from e
  728. if not ruamel.yaml.__version__:
  729. raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
  730. m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
  731. ruamel.yaml.__version__)
  732. if not m:
  733. raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
  734. if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
  735. raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
  736. # suiteable version ruamel.yaml imported successfully
  737. _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
  738. 'true', 'True', 'TRUE',
  739. 'on', 'On', 'ON')
  740. _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
  741. 'false', 'False', 'FALSE',
  742. 'off', 'Off', 'OFF')
  743. _yaml_bools_plus_lists = []
  744. _yaml_bools_plus_lists.extend(_yaml_bools_true)
  745. _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
  746. _yaml_bools_plus_lists.extend(_yaml_bools_false)
  747. _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
  748. def _class_as_dict_representer(dumper, data):
  749. '''Creates a YAML representation of a App/Build instance'''
  750. return dumper.represent_dict(data)
  751. def _field_to_yaml(typ, value):
  752. if typ is TYPE_STRING:
  753. if value in _yaml_bools_plus_lists:
  754. return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
  755. return str(value)
  756. elif typ is TYPE_INT:
  757. return int(value)
  758. elif typ is TYPE_MULTILINE:
  759. if '\n' in value:
  760. return ruamel.yaml.scalarstring.preserve_literal(str(value))
  761. else:
  762. return str(value)
  763. elif typ is TYPE_SCRIPT:
  764. if type(value) == list:
  765. if len(value) == 1:
  766. return value[0]
  767. else:
  768. return value
  769. else:
  770. script_lines = value.split(' && ')
  771. if len(script_lines) > 1:
  772. return script_lines
  773. else:
  774. return value
  775. else:
  776. return value
  777. def _app_to_yaml(app):
  778. cm = ruamel.yaml.comments.CommentedMap()
  779. insert_newline = False
  780. for field in yaml_app_field_order:
  781. if field == '\n':
  782. # next iteration will need to insert a newline
  783. insert_newline = True
  784. else:
  785. if app.get(field) or field == 'Builds':
  786. if field == 'Builds':
  787. if app.get('Builds'):
  788. cm.update({field: _builds_to_yaml(app)})
  789. elif field == 'CurrentVersionCode':
  790. cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
  791. else:
  792. cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
  793. if insert_newline:
  794. # we need to prepend a newline in front of this field
  795. insert_newline = False
  796. # inserting empty lines is not supported so we add a
  797. # bogus comment and over-write its value
  798. cm.yaml_set_comment_before_after_key(field, 'bogus')
  799. cm.ca.items[field][1][-1].value = '\n'
  800. return cm
  801. def _builds_to_yaml(app):
  802. builds = ruamel.yaml.comments.CommentedSeq()
  803. for build in app.get('Builds', []):
  804. if not isinstance(build, Build):
  805. build = Build(build)
  806. b = ruamel.yaml.comments.CommentedMap()
  807. for field in build_flags:
  808. value = getattr(build, field)
  809. if hasattr(build, field) and value:
  810. if field == 'gradle' and value == ['off']:
  811. value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
  812. if field in ('maven', 'buildozer'):
  813. if value == 'no':
  814. continue
  815. elif value == 'yes':
  816. value = 'yes'
  817. b.update({field: _field_to_yaml(flagtype(field), value)})
  818. builds.append(b)
  819. # insert extra empty lines between build entries
  820. for i in range(1, len(builds)):
  821. builds.yaml_set_comment_before_after_key(i, 'bogus')
  822. builds.ca.items[i][1][-1].value = '\n'
  823. return builds
  824. yaml_app = _app_to_yaml(app)
  825. ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
  826. build_line_sep = re.compile(r'(?<!\\),')
  827. build_cont = re.compile(r'^[ \t]')
  828. def write_metadata(metadatapath, app):
  829. if metadatapath.endswith('.yml'):
  830. if importlib.util.find_spec('ruamel.yaml'):
  831. with open(metadatapath, 'w') as mf:
  832. return write_yaml(mf, app)
  833. else:
  834. raise FDroidException(_('ruamel.yaml not installed, can not write metadata.'))
  835. _warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
  836. def add_metadata_arguments(parser):
  837. '''add common command line flags related to metadata processing'''
  838. parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
  839. help=_("force metadata errors (default) to be warnings, or to be ignored."))