PageRenderTime 29ms CodeModel.GetById 16ms RepoModel.GetById 0ms app.codeStats 0ms

/fdroidserver/scanner.py

https://gitlab.com/dj-tech/fdroidserver
Python | 329 lines | 265 code | 40 blank | 24 comment | 53 complexity | 6bce77173a3e773ab2739687a97ccae5 MD5 | raw file
  1. #!/usr/bin/env python3
  2. #
  3. # scanner.py - part of the FDroid server tools
  4. # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
  5. #
  6. # This program is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU Affero General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU Affero General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Affero General Public License
  17. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  18. import os
  19. import re
  20. import traceback
  21. from argparse import ArgumentParser
  22. import logging
  23. from . import common
  24. from . import metadata
  25. from .common import BuildException, VCSException
  26. config = None
  27. options = None
  28. def get_gradle_compile_commands(build):
  29. compileCommands = ['compile', 'releaseCompile']
  30. if build.gradle and build.gradle != ['yes']:
  31. compileCommands += [flavor + 'Compile' for flavor in build.gradle]
  32. compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
  33. return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
  34. # Scan the source code in the given directory (and all subdirectories)
  35. # and return the number of fatal problems encountered
  36. def scan_source(build_dir, root_dir, build):
  37. count = 0
  38. # Common known non-free blobs (always lower case):
  39. usual_suspects = {
  40. exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
  41. r'flurryagent',
  42. r'paypal.*mpl',
  43. r'google.*analytics',
  44. r'admob.*sdk.*android',
  45. r'google.*ad.*view',
  46. r'google.*admob',
  47. r'google.*play.*services',
  48. r'crittercism',
  49. r'heyzap',
  50. r'jpct.*ae',
  51. r'youtube.*android.*player.*api',
  52. r'bugsense',
  53. r'crashlytics',
  54. r'ouya.*sdk',
  55. r'libspen23',
  56. ]
  57. }
  58. def suspects_found(s):
  59. for n, r in usual_suspects.items():
  60. if r.match(s):
  61. yield n
  62. gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
  63. allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
  64. 'repo1.maven.org/maven2', # mavenCentral()
  65. 'jcenter.bintray.com', # jcenter()
  66. 'jitpack.io',
  67. 'repo.maven.apache.org/maven2',
  68. 'oss.sonatype.org/content/repositories/snapshots',
  69. 'oss.sonatype.org/content/repositories/releases',
  70. 'oss.sonatype.org/content/groups/public',
  71. 'clojars.org/repo', # Clojure free software libs
  72. 's3.amazonaws.com/repo.commonsware.com', # CommonsWare
  73. 'plugins.gradle.org/m2', # Gradle plugin repo
  74. ]
  75. ]
  76. scanignore = common.getpaths_map(build_dir, build.scanignore)
  77. scandelete = common.getpaths_map(build_dir, build.scandelete)
  78. scanignore_worked = set()
  79. scandelete_worked = set()
  80. def toignore(fd):
  81. for k, paths in scanignore.items():
  82. for p in paths:
  83. if fd.startswith(p):
  84. scanignore_worked.add(k)
  85. return True
  86. return False
  87. def todelete(fd):
  88. for k, paths in scandelete.items():
  89. for p in paths:
  90. if fd.startswith(p):
  91. scandelete_worked.add(k)
  92. return True
  93. return False
  94. def ignoreproblem(what, fd, fp):
  95. logging.info('Ignoring %s at %s' % (what, fd))
  96. return 0
  97. def removeproblem(what, fd, fp):
  98. logging.info('Removing %s at %s' % (what, fd))
  99. os.remove(fp)
  100. return 0
  101. def warnproblem(what, fd):
  102. if toignore(fd):
  103. return
  104. logging.warn('Found %s at %s' % (what, fd))
  105. def handleproblem(what, fd, fp):
  106. if toignore(fd):
  107. return ignoreproblem(what, fd, fp)
  108. if todelete(fd):
  109. return removeproblem(what, fd, fp)
  110. logging.error('Found %s at %s' % (what, fd))
  111. return 1
  112. def is_executable(path):
  113. return os.path.exists(path) and os.access(path, os.X_OK)
  114. textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
  115. def is_binary(path):
  116. d = None
  117. with open(path, 'rb') as f:
  118. d = f.read(1024)
  119. return bool(d.translate(None, textchars))
  120. # False positives patterns for files that are binary and executable.
  121. safe_paths = [re.compile(r) for r in [
  122. r".*/drawable[^/]*/.*\.png$", # png drawables
  123. r".*/mipmap[^/]*/.*\.png$", # png mipmaps
  124. ]
  125. ]
  126. def safe_path(path):
  127. for sp in safe_paths:
  128. if sp.match(path):
  129. return True
  130. return False
  131. gradle_compile_commands = get_gradle_compile_commands(build)
  132. def is_used_by_gradle(line):
  133. return any(command.match(line) for command in gradle_compile_commands)
  134. # Iterate through all files in the source code
  135. for r, d, f in os.walk(build_dir, topdown=True):
  136. # It's topdown, so checking the basename is enough
  137. for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
  138. if ignoredir in d:
  139. d.remove(ignoredir)
  140. for curfile in f:
  141. if curfile in ['.DS_Store']:
  142. continue
  143. # Path (relative) to the file
  144. fp = os.path.join(r, curfile)
  145. if os.path.islink(fp):
  146. continue
  147. fd = fp[len(build_dir) + 1:]
  148. _, ext = common.get_extension(fd)
  149. if ext == 'so':
  150. count += handleproblem('shared library', fd, fp)
  151. elif ext == 'a':
  152. count += handleproblem('static library', fd, fp)
  153. elif ext == 'class':
  154. count += handleproblem('Java compiled class', fd, fp)
  155. elif ext == 'apk':
  156. removeproblem('APK file', fd, fp)
  157. elif ext == 'jar':
  158. for name in suspects_found(curfile):
  159. count += handleproblem('usual supect \'%s\'' % name, fd, fp)
  160. warnproblem('JAR file', fd)
  161. elif ext == 'java':
  162. if not os.path.isfile(fp):
  163. continue
  164. with open(fp, 'r', encoding='utf8') as f:
  165. for line in f:
  166. if 'DexClassLoader' in line:
  167. count += handleproblem('DexClassLoader', fd, fp)
  168. break
  169. elif ext == 'gradle':
  170. if not os.path.isfile(fp):
  171. continue
  172. with open(fp, 'r', encoding='utf8') as f:
  173. lines = f.readlines()
  174. for i, line in enumerate(lines):
  175. if is_used_by_gradle(line):
  176. for name in suspects_found(line):
  177. count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), fd, fp)
  178. noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
  179. joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
  180. for m in gradle_mavenrepo.finditer(joined):
  181. url = m.group(2)
  182. if not any(r.match(url) for r in allowed_repos):
  183. count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp)
  184. elif ext in ['', 'bin', 'out', 'exe']:
  185. if is_binary(fp):
  186. count += handleproblem('binary', fd, fp)
  187. elif is_executable(fp):
  188. if is_binary(fp) and not safe_path(fd):
  189. warnproblem('possible binary', fd)
  190. for p in scanignore:
  191. if p not in scanignore_worked:
  192. logging.error('Unused scanignore path: %s' % p)
  193. count += 1
  194. for p in scandelete:
  195. if p not in scandelete_worked:
  196. logging.error('Unused scandelete path: %s' % p)
  197. count += 1
  198. return count
  199. def main():
  200. global config, options
  201. # Parse command line...
  202. parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
  203. common.setup_global_opts(parser)
  204. parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
  205. metadata.add_metadata_arguments(parser)
  206. options = parser.parse_args()
  207. metadata.warnings_action = options.W
  208. config = common.read_config(options)
  209. # Read all app and srclib metadata
  210. allapps = metadata.read_metadata()
  211. apps = common.read_app_args(options.appid, allapps, True)
  212. probcount = 0
  213. build_dir = 'build'
  214. if not os.path.isdir(build_dir):
  215. logging.info("Creating build directory")
  216. os.makedirs(build_dir)
  217. srclib_dir = os.path.join(build_dir, 'srclib')
  218. extlib_dir = os.path.join(build_dir, 'extlib')
  219. for appid, app in apps.items():
  220. if app.Disabled:
  221. logging.info("Skipping %s: disabled" % appid)
  222. continue
  223. if not app.builds:
  224. logging.info("Skipping %s: no builds specified" % appid)
  225. continue
  226. logging.info("Processing " + appid)
  227. try:
  228. if app.RepoType == 'srclib':
  229. build_dir = os.path.join('build', 'srclib', app.Repo)
  230. else:
  231. build_dir = os.path.join('build', appid)
  232. # Set up vcs interface and make sure we have the latest code...
  233. vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
  234. for build in app.builds:
  235. if build.disable:
  236. logging.info("...skipping version %s - %s" % (
  237. build.version, build.get('disable', build.commit[1:])))
  238. else:
  239. logging.info("...scanning version " + build.version)
  240. # Prepare the source code...
  241. root_dir, _ = common.prepare_source(vcs, app, build,
  242. build_dir, srclib_dir,
  243. extlib_dir, False)
  244. # Do the scan...
  245. count = scan_source(build_dir, root_dir, build)
  246. if count > 0:
  247. logging.warn('Scanner found %d problems in %s (%s)' % (
  248. count, appid, build.vercode))
  249. probcount += count
  250. except BuildException as be:
  251. logging.warn("Could not scan app %s due to BuildException: %s" % (
  252. appid, be))
  253. probcount += 1
  254. except VCSException as vcse:
  255. logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
  256. probcount += 1
  257. except Exception:
  258. logging.warn("Could not scan app %s due to unknown error: %s" % (
  259. appid, traceback.format_exc()))
  260. probcount += 1
  261. logging.info("Finished:")
  262. print("%d problems found" % probcount)
  263. if __name__ == "__main__":
  264. main()