/run_comparison.py
Python | 308 lines | 231 code | 33 blank | 44 comment | 9 complexity | 98d805be253f5c05aedd7dc50f87b15c MD5 | raw file
- #!/usr/bin/env python
- # This file provided by Facebook is for non-commercial testing and evaluation
- # purposes only. Facebook reserves all rights not expressly granted.
- #
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- # FACEBOOK BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- """
- This script builds and runs the comparison app, switching from one library to the next,
- taking measurements as it goes, and outputs the results neatly.
- Due to a bug, you must specify the CPU when running the script.
- Use -c armeabi-v7a for most phones. Use -c armeabi for ARM v5-6 phones, or
- -c arm64 for 64-bit ARM devices. Some emulators and tablets will need -c x86.
- To select a subset of the libraries, use the -s option with a
- space-separated list. Available options are fresco, fresco-okhttp,
- glide, volley, drawee-volley, uil, picasso, and aquery.
- To see the comparison for only network or local images, use -d network or -d local.
- Note that Volley does not support local images, and fresco and fresco-okhttp
- are identical for local images.
- Results will vary based on the the device, the network conditions and the mix of images available.
- Example: to run a local-only comparison of fresco and picasso on an ARM v7 device:
- ./run_comparison.py -s fresco picasso -d local -c armeabi-v7a
- """
- from __future__ import absolute_import
- from __future__ import division
- from __future__ import print_function
- from __future__ import unicode_literals
- import argparse
- import glob
- import os
- import re
- import tempfile
- from collections import namedtuple
- from subprocess import check_call, PIPE, Popen
- """ List of tested libraries """
- TESTS = (
- 'fresco',
- 'fresco-okhttp',
- 'glide',
- 'picasso',
- 'uil',
- 'volley',
- 'drawee-volley',
- 'aquery',
- )
- TEST_SOURCES = (
- 'network',
- 'local'
- )
- ABIS = (
- 'arm64-v8a',
- 'armeabi',
- 'armeabi-v7a',
- 'x86',
- 'x86_64'
- )
- """ Appends test class name to method name """
- TEST_PATTERN = 'test{}{}'
- """ Named tuple containing relevant numbers reported by a test """
- Stats = namedtuple('Stats', [
- 'success_wait_times',
- 'failure_wait_times',
- 'cancellation_wait_times',
- 'java_heap_sizes',
- 'native_heap_sizes',
- 'skipped_frames'])
- def parse_args():
- parser = argparse.ArgumentParser(
- description='Runs comparison test and processes results')
- parser.add_argument('-s', '--scenarios', choices=TESTS, nargs='+')
- parser.add_argument('-d', '--sources', choices=TEST_SOURCES, nargs='+')
- parser.add_argument('-c', '--cpu', choices=ABIS, required=True)
- return parser.parse_args()
- def start_subprocess(command, **kwargs):
- """ Starts subprocess after printing command to stdout. """
- return Popen(command.split(), **kwargs)
- def run_command(command):
- """ Runs given command and waits for it to terminate.
- Prints the command to stdout and redirects its output to /dev/null. """
- with open('/dev/null', 'w') as devnull:
- check_call(command.split(), stdout=devnull, stderr=devnull)
- def gradle(*tasks):
- """ Runs given gradle tasks """
- if tasks:
- run_command('./gradlew {}'.format(" ".join(tasks)))
- def adb(command):
- """ Runs adb command - arguments are given as single string"""
- run_command('adb {}'.format(command))
- def install_apks(abi):
- """ Installs comparison app and test apks """
- print("Installing comparison app...")
- gradle(':samples:comparison:assembleDebug',
- ':samples:comparison:assembleDebugAndroidTest')
- adb('uninstall com.facebook.samples.comparison')
- adb('uninstall com.facebook.samples.comparison.test')
- cmd = ('install -r samples/comparison/build/outputs/apk/comparison-'
- '{}-debug.apk'.format(abi))
- adb(cmd)
- adb('install -r samples/comparison/build/outputs/apk/'
- 'comparison-debug-androidTest-unaligned.apk')
- class ComparisonTest:
- """ Comparison test case """
- def __init__(
- self,
- method_name,
- class_name='com.facebook.samples.comparison.test.ScrollTest',
- test_package='com.facebook.samples.comparison.test',
- test_runner='android.test.InstrumentationTestRunner'):
- self.method_name = method_name
- self.class_name = class_name
- self.test_package = test_package
- self.test_runner = test_runner
- def __call__(self):
- """ Executes test case and captures logcat output """
- adb('logcat -c')
- with tempfile.TemporaryFile() as logcat_file:
- logcat_reader = start_subprocess(
- 'adb logcat',
- stdout=logcat_file)
- adb('shell am instrument -w -e class {}#{} {}/{}'.format(
- self.class_name,
- self.method_name,
- self.test_package,
- self.test_runner))
- logcat_reader.terminate()
- logcat_reader.wait()
- logcat_file.seek(0)
- self.logcat = logcat_file.readlines()
- def get_float_from_logs(regex, logs):
- pattern = re.compile(regex)
- return [float(match.group(1)) for match in map(pattern.search, logs) if match]
- def get_int_from_logs(regex, logs):
- pattern = re.compile(regex)
- return [int(match.group(1)) for match in map(pattern.search, logs) if match]
- def get_stats(logs):
- pattern = re.compile("""]: loaded after (\d+) ms""")
- success_wait_times = [
- int(match.group(1)) for match in map(pattern.search, logs) if match]
- pattern = re.compile("""]: failed after (\d+) ms""")
- failure_wait_times = [
- int(match.group(1)) for match in map(pattern.search, logs) if match]
- pattern = re.compile("""]: cancelled after (\d+) ms""")
- cancellation_wait_times = [
- int(match.group(1)) for match in map(pattern.search, logs) if match]
- pattern = re.compile("""\s+(\d+.\d+) MB Java""")
- java_heap_sizes = [
- float(match.group(1)) for match in map(pattern.search, logs) if match]
- pattern = re.compile("""\s+(\d+.\d+) MB native""")
- native_heap_sizes = [
- float(match.group(1)) for match in map(pattern.search, logs) if match]
- pattern = re.compile("""Skipped (\d+) frames! The application may be""")
- skipped_frames = [
- int(match.group(1)) for match in map(pattern.search, logs) if match]
- return Stats(
- success_wait_times,
- failure_wait_times,
- cancellation_wait_times,
- java_heap_sizes,
- native_heap_sizes,
- skipped_frames)
- def print_stats(stats):
- successes = len(stats.success_wait_times)
- cancellations = len(stats.cancellation_wait_times)
- failures = len(stats.failure_wait_times)
- total_count = successes + cancellations + failures
- if total_count == 0:
- print("Unable to read logs.")
- return
- total_wait_time = (
- sum(stats.success_wait_times) +
- sum(stats.cancellation_wait_times) +
- sum(stats.failure_wait_times))
- avg_wait_time = float(total_wait_time) / total_count
- max_java_heap = max(stats.java_heap_sizes)
- max_native_heap = max(stats.native_heap_sizes)
- total_skipped_frames = sum(stats.skipped_frames)
- print("Average wait time = {0:.1f}".format(avg_wait_time))
- print("Successful requests = {}".format(successes))
- print("Failures = {}".format(failures))
- print("Cancellations = {}".format(cancellations))
- print("Max java heap = {0:.1f}".format(max_java_heap))
- print("Max native heap = {0:.1f}".format(max_native_heap))
- print("Total skipped frames = {}".format(total_skipped_frames))
- def get_test_name(option_name, source_name):
- return TEST_PATTERN.format(
- ''.join(word.capitalize() for word in option_name.split('-')), source_name.capitalize())
- def valid_scenario(scenario_name, source_name):
- return source_name != 'local' or (scenario_name != 'volley' and scenario_name != 'drawee-volley')
- def list_producers():
- sdir = os.path.dirname(os.path.abspath(__file__))
- producer_path = '%s/imagepipeline/src/main/java/com/facebook/imagepipeline/producers/*Producer.java' % sdir
- files = glob.glob(producer_path)
- return [f.split('.')[0].split('/')[-1] for f in files]
- def print_fresco_perf_line(margin, name, times):
- length = len(times)
- if length == 0:
- return
- print("%s: %d requests, avg %d" % (name.rjust(margin), length, float(sum(times)) / length))
- def print_fresco_perf(logs):
- producers = list_producers()
- margin = max([len(p) for p in producers])
- requests = get_int_from_logs(""".*RequestLoggingListener.*onRequestSuccess.*elapsedTime:\s(\d+).*""", logs)
- print_fresco_perf_line(margin, 'Total', requests)
- for producer in producers:
- queue = get_int_from_logs(".*onProducerFinishWithSuccess.*producer:\s%s.*queueTime=(\d+).*" % producer, logs)
- print_fresco_perf_line(margin, '%s queue' % producer, queue)
- times = get_int_from_logs(".*onProducerFinishWithSuccess.*producer:\s%s.*elapsedTime:\s(\d+).*" % producer, logs)
- print_fresco_perf_line(margin, producer, times)
- def main():
- args = parse_args()
- scenarios = []
- sources = []
- if args.scenarios:
- scenarios = args.scenarios
- else:
- scenarios = TESTS
- if args.sources:
- sources = args.sources
- else:
- sources = TEST_SOURCES
- install_apks(args.cpu)
- for source_name in sources:
- for scenario_name in scenarios:
- if valid_scenario(scenario_name, source_name):
- print()
- print('Testing {} {}'.format(scenario_name, source_name))
- print(get_test_name(scenario_name, source_name))
- test = ComparisonTest(get_test_name(scenario_name, source_name))
- test()
- stats = get_stats(test.logcat)
- print_stats(stats)
- if scenario_name[:6] == 'fresco':
- print()
- print_fresco_perf(test.logcat)
- if __name__ == "__main__":
- main()