PageRenderTime 63ms CodeModel.GetById 44ms app.highlight 16ms RepoModel.GetById 0ms app.codeStats 0ms

/fdroidserver/scanner.py

https://gitlab.com/wrafal/fdroidserver
Python | 329 lines | 265 code | 40 blank | 24 comment | 75 complexity | 6bce77173a3e773ab2739687a97ccae5 MD5 | raw file
  1#!/usr/bin/env python3
  2#
  3# scanner.py - part of the FDroid server tools
  4# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
  5#
  6# This program is free software: you can redistribute it and/or modify
  7# it under the terms of the GNU Affero General Public License as published by
  8# the Free Software Foundation, either version 3 of the License, or
  9# (at your option) any later version.
 10#
 11# This program is distributed in the hope that it will be useful,
 12# but WITHOUT ANY WARRANTY; without even the implied warranty of
 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 14# GNU Affero General Public License for more details.
 15#
 16# You should have received a copy of the GNU Affero General Public License
 17# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 18
 19import os
 20import re
 21import traceback
 22from argparse import ArgumentParser
 23import logging
 24
 25from . import common
 26from . import metadata
 27from .common import BuildException, VCSException
 28
 29config = None
 30options = None
 31
 32
 33def get_gradle_compile_commands(build):
 34    compileCommands = ['compile', 'releaseCompile']
 35    if build.gradle and build.gradle != ['yes']:
 36        compileCommands += [flavor + 'Compile' for flavor in build.gradle]
 37        compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
 38
 39    return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
 40
 41
 42# Scan the source code in the given directory (and all subdirectories)
 43# and return the number of fatal problems encountered
 44def scan_source(build_dir, root_dir, build):
 45
 46    count = 0
 47
 48    # Common known non-free blobs (always lower case):
 49    usual_suspects = {
 50        exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
 51            r'flurryagent',
 52            r'paypal.*mpl',
 53            r'google.*analytics',
 54            r'admob.*sdk.*android',
 55            r'google.*ad.*view',
 56            r'google.*admob',
 57            r'google.*play.*services',
 58            r'crittercism',
 59            r'heyzap',
 60            r'jpct.*ae',
 61            r'youtube.*android.*player.*api',
 62            r'bugsense',
 63            r'crashlytics',
 64            r'ouya.*sdk',
 65            r'libspen23',
 66        ]
 67    }
 68
 69    def suspects_found(s):
 70        for n, r in usual_suspects.items():
 71            if r.match(s):
 72                yield n
 73
 74    gradle_mavenrepo = re.compile(r'maven *{ *(url)? *[\'"]?([^ \'"]*)[\'"]?')
 75
 76    allowed_repos = [re.compile(r'^https?://' + re.escape(repo) + r'/*') for repo in [
 77        'repo1.maven.org/maven2',  # mavenCentral()
 78        'jcenter.bintray.com',     # jcenter()
 79        'jitpack.io',
 80        'repo.maven.apache.org/maven2',
 81        'oss.sonatype.org/content/repositories/snapshots',
 82        'oss.sonatype.org/content/repositories/releases',
 83        'oss.sonatype.org/content/groups/public',
 84        'clojars.org/repo',  # Clojure free software libs
 85        's3.amazonaws.com/repo.commonsware.com',  # CommonsWare
 86        'plugins.gradle.org/m2',  # Gradle plugin repo
 87        ]
 88    ]
 89
 90    scanignore = common.getpaths_map(build_dir, build.scanignore)
 91    scandelete = common.getpaths_map(build_dir, build.scandelete)
 92
 93    scanignore_worked = set()
 94    scandelete_worked = set()
 95
 96    def toignore(fd):
 97        for k, paths in scanignore.items():
 98            for p in paths:
 99                if fd.startswith(p):
100                    scanignore_worked.add(k)
101                    return True
102        return False
103
104    def todelete(fd):
105        for k, paths in scandelete.items():
106            for p in paths:
107                if fd.startswith(p):
108                    scandelete_worked.add(k)
109                    return True
110        return False
111
112    def ignoreproblem(what, fd, fp):
113        logging.info('Ignoring %s at %s' % (what, fd))
114        return 0
115
116    def removeproblem(what, fd, fp):
117        logging.info('Removing %s at %s' % (what, fd))
118        os.remove(fp)
119        return 0
120
121    def warnproblem(what, fd):
122        if toignore(fd):
123            return
124        logging.warn('Found %s at %s' % (what, fd))
125
126    def handleproblem(what, fd, fp):
127        if toignore(fd):
128            return ignoreproblem(what, fd, fp)
129        if todelete(fd):
130            return removeproblem(what, fd, fp)
131        logging.error('Found %s at %s' % (what, fd))
132        return 1
133
134    def is_executable(path):
135        return os.path.exists(path) and os.access(path, os.X_OK)
136
137    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
138
139    def is_binary(path):
140        d = None
141        with open(path, 'rb') as f:
142            d = f.read(1024)
143        return bool(d.translate(None, textchars))
144
145    # False positives patterns for files that are binary and executable.
146    safe_paths = [re.compile(r) for r in [
147        r".*/drawable[^/]*/.*\.png$",  # png drawables
148        r".*/mipmap[^/]*/.*\.png$",    # png mipmaps
149        ]
150    ]
151
152    def safe_path(path):
153        for sp in safe_paths:
154            if sp.match(path):
155                return True
156        return False
157
158    gradle_compile_commands = get_gradle_compile_commands(build)
159
160    def is_used_by_gradle(line):
161        return any(command.match(line) for command in gradle_compile_commands)
162
163    # Iterate through all files in the source code
164    for r, d, f in os.walk(build_dir, topdown=True):
165
166        # It's topdown, so checking the basename is enough
167        for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
168            if ignoredir in d:
169                d.remove(ignoredir)
170
171        for curfile in f:
172
173            if curfile in ['.DS_Store']:
174                continue
175
176            # Path (relative) to the file
177            fp = os.path.join(r, curfile)
178
179            if os.path.islink(fp):
180                continue
181
182            fd = fp[len(build_dir) + 1:]
183            _, ext = common.get_extension(fd)
184
185            if ext == 'so':
186                count += handleproblem('shared library', fd, fp)
187            elif ext == 'a':
188                count += handleproblem('static library', fd, fp)
189            elif ext == 'class':
190                count += handleproblem('Java compiled class', fd, fp)
191            elif ext == 'apk':
192                removeproblem('APK file', fd, fp)
193
194            elif ext == 'jar':
195                for name in suspects_found(curfile):
196                    count += handleproblem('usual supect \'%s\'' % name, fd, fp)
197                warnproblem('JAR file', fd)
198
199            elif ext == 'java':
200                if not os.path.isfile(fp):
201                    continue
202                with open(fp, 'r', encoding='utf8') as f:
203                    for line in f:
204                        if 'DexClassLoader' in line:
205                            count += handleproblem('DexClassLoader', fd, fp)
206                            break
207
208            elif ext == 'gradle':
209                if not os.path.isfile(fp):
210                    continue
211                with open(fp, 'r', encoding='utf8') as f:
212                    lines = f.readlines()
213                for i, line in enumerate(lines):
214                    if is_used_by_gradle(line):
215                        for name in suspects_found(line):
216                            count += handleproblem('usual supect \'%s\' at line %d' % (name, i + 1), fd, fp)
217                noncomment_lines = [l for l in lines if not common.gradle_comment.match(l)]
218                joined = re.sub(r'[\n\r\s]+', ' ', ' '.join(noncomment_lines))
219                for m in gradle_mavenrepo.finditer(joined):
220                    url = m.group(2)
221                    if not any(r.match(url) for r in allowed_repos):
222                        count += handleproblem('unknown maven repo \'%s\'' % url, fd, fp)
223
224            elif ext in ['', 'bin', 'out', 'exe']:
225                if is_binary(fp):
226                    count += handleproblem('binary', fd, fp)
227
228            elif is_executable(fp):
229                if is_binary(fp) and not safe_path(fd):
230                    warnproblem('possible binary', fd)
231
232    for p in scanignore:
233        if p not in scanignore_worked:
234            logging.error('Unused scanignore path: %s' % p)
235            count += 1
236
237    for p in scandelete:
238        if p not in scandelete_worked:
239            logging.error('Unused scandelete path: %s' % p)
240            count += 1
241
242    return count
243
244
245def main():
246
247    global config, options
248
249    # Parse command line...
250    parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
251    common.setup_global_opts(parser)
252    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
253    metadata.add_metadata_arguments(parser)
254    options = parser.parse_args()
255    metadata.warnings_action = options.W
256
257    config = common.read_config(options)
258
259    # Read all app and srclib metadata
260    allapps = metadata.read_metadata()
261    apps = common.read_app_args(options.appid, allapps, True)
262
263    probcount = 0
264
265    build_dir = 'build'
266    if not os.path.isdir(build_dir):
267        logging.info("Creating build directory")
268        os.makedirs(build_dir)
269    srclib_dir = os.path.join(build_dir, 'srclib')
270    extlib_dir = os.path.join(build_dir, 'extlib')
271
272    for appid, app in apps.items():
273
274        if app.Disabled:
275            logging.info("Skipping %s: disabled" % appid)
276            continue
277        if not app.builds:
278            logging.info("Skipping %s: no builds specified" % appid)
279            continue
280
281        logging.info("Processing " + appid)
282
283        try:
284
285            if app.RepoType == 'srclib':
286                build_dir = os.path.join('build', 'srclib', app.Repo)
287            else:
288                build_dir = os.path.join('build', appid)
289
290            # Set up vcs interface and make sure we have the latest code...
291            vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
292
293            for build in app.builds:
294
295                if build.disable:
296                    logging.info("...skipping version %s - %s" % (
297                        build.version, build.get('disable', build.commit[1:])))
298                else:
299                    logging.info("...scanning version " + build.version)
300
301                    # Prepare the source code...
302                    root_dir, _ = common.prepare_source(vcs, app, build,
303                                                        build_dir, srclib_dir,
304                                                        extlib_dir, False)
305
306                    # Do the scan...
307                    count = scan_source(build_dir, root_dir, build)
308                    if count > 0:
309                        logging.warn('Scanner found %d problems in %s (%s)' % (
310                            count, appid, build.vercode))
311                        probcount += count
312
313        except BuildException as be:
314            logging.warn("Could not scan app %s due to BuildException: %s" % (
315                appid, be))
316            probcount += 1
317        except VCSException as vcse:
318            logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
319            probcount += 1
320        except Exception:
321            logging.warn("Could not scan app %s due to unknown error: %s" % (
322                appid, traceback.format_exc()))
323            probcount += 1
324
325    logging.info("Finished:")
326    print("%d problems found" % probcount)
327
328if __name__ == "__main__":
329    main()