| #!/usr/bin/env python |
| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be found |
| # in the LICENSE file. |
| |
| """ Analyze recent bench data from graphs, and output suggested ranges. |
| |
| This script reads and parses Skia benchmark values from the xhtml files |
| generated by bench_graph_svg.py, and outputs an html file containing suggested |
| bench ranges to use in bench_expectations.txt, with analytical plots. |
| """ |
| |
| __author__ = 'bensong@google.com (Ben Chen)' |
| |
| import getopt |
| import math |
| import re |
| import sys |
| import urllib |
| from datetime import datetime |
| |
| |
| # Constants for calculating suggested bench ranges. |
| WINDOW = 5 # Moving average sliding window size. |
| # We use moving average as expected bench value, and calculate average Variance |
| # of bench from the moving average. Set range to be [X_UB * Variance above |
| # moving average, X_LB * Variance below moving average] of latest revision. |
| X_UB = 4.0 |
| X_LB = 5.0 |
| |
| # List of platforms. |
| PLATFORMS = ['GalaxyNexus_4-1_Float_Release', |
| 'Mac_Float_NoDebug_32', |
| 'Mac_Float_NoDebug_64', |
| 'MacMiniLion_Float_NoDebug_32', |
| 'MacMiniLion_Float_NoDebug_64', |
| 'Nexus7_4-1_Float_Release', |
| 'Shuttle_Ubuntu12_ATI5770_Float_Release_64', |
| 'Shuttle_Win7_Intel_Float_Release_32', |
| 'Shuttle_Win7_Intel_Float_Release_64', |
| 'Xoom_4-1_Float_Release' |
| ] |
| |
| # List of bench representation algorithms. Flag "-a" is chosen from the list. |
| ALGS = ['25th', 'avg', 'med', 'min'] |
| |
| # Regular expressions for parsing bench/revision values. |
| HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->' |
| REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="' # Revision corresponding x. |
| LINE_RE = '<polyline id="(.*)".*points="(.*)"/>' # Bench value lines. |
| |
| # Bench graph url pattern. |
| INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com' |
| '/graph-Skia_%s-2.xhtml') |
| |
| # Output HTML elements and templates. |
| HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>' |
| '<script type="text/javascript" src="https://skia.googlecode.com/' |
| 'svn/buildbot/dygraph-combined.js"></script></head><body>Please ' |
| 'adjust values as appropriate and update benches to monitor in ' |
| 'bench/bench_expectations.txt.<br><br>') |
| HTML_SUFFIX = '</body></html>' |
| GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>' |
| '<script type="text/javascript">g%s=new Dygraph(' |
| 'document.getElementById("%s"),"rev,bench,alert\\n') |
| GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,' |
| 'pointSize:4,highlightCircleSize:6}});</script>') |
| |
| |
| def Usage(): |
| """Prints flag usage information.""" |
| print '-a <representation-algorithm>: defaults to "25th".' |
| print ' If set, must be one of the list element in ALGS defined above.' |
| print '-b <bench-prefix>: prefix of matching bench names to analyze.' |
| print ' Only include benchmarks whose names start with this string.' |
| print ' Cannot be empty, because there are too many benches overall.' |
| print '-o <file>: html output filename. Output to STDOUT if not set.' |
| print '-p <platform-prefix>: prefix of platform names to analyze.' |
| print ' PLATFORMS has list of matching candidates. Matches all if not set.' |
| |
| def GetBenchValues(page, bench_prefix): |
| """Returns a dict of matching bench values from the given xhtml page. |
| Args: |
| page: substring used to construct the specific bench graph URL to fetch. |
| bench_prefix: only benches starting with this string will be included. |
| |
| Returns: |
| a dict mapping benchmark name and revision combinations to bench values. |
| """ |
| height = None |
| max_bench = None |
| height_scale = None |
| revisions = [] |
| x_axes = [] # For calculating corresponding revisions. |
| val_dic = {} # dict[bench_name][revision] -> bench_value |
| |
| lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines() |
| for line in lines: |
| height_match = re.search(HEIGHT_RE, line) |
| if height_match: |
| height = float(height_match.group(1)) |
| max_bench = float(height_match.group(2)) |
| height_scale = max_bench / height |
| |
| rev_match = re.search(REV_RE, line) |
| if rev_match: |
| revisions.append(int(rev_match.group(1))) |
| x_axes.append(float(rev_match.group(2))) |
| |
| line_match = re.search(LINE_RE, line) |
| if not line_match: |
| continue |
| bench = line_match.group(1) |
| bench = bench[:bench.find('_{')] |
| if not bench.startswith(bench_prefix): |
| continue |
| if bench not in val_dic: |
| val_dic[bench] = {} |
| |
| vals = line_match.group(2).strip().split(' ') |
| if len(vals) < WINDOW: # Too few bench data points; skip. |
| continue |
| for val in vals: |
| x, y = [float(i) for i in val.split(',')] |
| for i in range(len(x_axes)): |
| if x <= x_axes[i]: # Found corresponding bench revision. |
| break |
| val_dic[bench][revisions[i]] = float( |
| '%.3f' % ((height - y) * height_scale)) |
| |
| return val_dic |
| |
| def CreateBenchOutput(page, bench, val_dic): |
| """Returns output for the given page and bench data in dict. |
| Args: |
| page: substring of bench graph webpage, to indicate the bench platform. |
| bench: name of the benchmark to process. |
| val_dic: dict[bench_name][revision] -> bench_value. |
| |
| Returns: |
| string of html/javascript as part of the whole script output for the bench. |
| """ |
| revs = val_dic[bench].keys() |
| revs.sort() |
| # Uses moving average to calculate expected bench variance, then sets |
| # expectations and ranges accordingly. |
| variances = [] |
| moving_avgs = [] |
| points = [] |
| for rev in revs: |
| points.append(val_dic[bench][rev]) |
| if len(points) >= WINDOW: |
| moving_avgs.append(sum(points[-WINDOW:]) / WINDOW) |
| variances.append(abs(points[-1] - moving_avgs[-1])) |
| else: # For the first WINDOW-1 points, cannot calculate moving average. |
| moving_avgs.append(points[-1]) # Uses actual value as estimates. |
| variances.append(0) |
| if len(variances) >= WINDOW: |
| for i in range(WINDOW - 1): |
| # Backfills estimated variances for the first WINDOW-1 points. |
| variances[i] = variances[WINDOW - 1] |
| |
| avg_var = sum(variances) / len(variances) |
| for val in variances: # Removes outlier variances. Only does one iter. |
| if val > min(X_LB, X_UB) * avg_var: |
| variances.remove(val) |
| avg_var = sum(variances) / len(variances) |
| |
| graph_id = '%s_%s' % (bench, page.replace('-', '_')) |
| expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1], |
| moving_avgs[-1] - X_LB * avg_var, |
| moving_avgs[-1] + X_UB * avg_var) |
| out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id) |
| for i in range(len(revs)): |
| out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var, |
| points[i], moving_avgs[i] + X_UB * avg_var) |
| if (points[i] > moving_avgs[i] + X_UB * avg_var or |
| points[i] < moving_avgs[i] - X_LB * avg_var): # Mark as alert point. |
| out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i]) |
| else: |
| out += 'NaN;NaN;NaN\\n' |
| |
| return out |
| |
| def main(): |
| """Parses flags and outputs analysis results.""" |
| try: |
| opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:') |
| except getopt.GetoptError, err: |
| Usage() |
| sys.exit(2) |
| |
| alg = '25th' |
| bench_prefix = None |
| out_file = None |
| platform_prefix = '' |
| for option, value in opts: |
| if option == '-a': |
| if value not in ALGS: |
| raise Exception('Invalid flag -a (%s): must be set to one of %s.' % |
| (value, str(ALGS))) |
| alg = value |
| elif option == '-b': |
| bench_prefix = value |
| elif option == '-o': |
| out_file = value |
| elif option == '-p': |
| platform_prefix = value |
| else: |
| Usage() |
| raise Exception('Error handling flags.') |
| |
| if not bench_prefix: |
| raise Exception('Must provide nonempty Flag -b (bench name prefix).') |
| |
| pages = [] |
| for platform in PLATFORMS: |
| if not platform.startswith(platform_prefix): |
| continue |
| pages.append('%s-%s' % (platform, alg)) |
| |
| if not pages: # No matching platform found. |
| raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' % |
| (platform_prefix, str(PLATFORMS))) |
| |
| body = '' |
| # Iterates through bench graph xhtml pages for oututting matching benches. |
| for page in pages: |
| bench_value_dict = GetBenchValues(page, bench_prefix) |
| for bench in bench_value_dict: |
| body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX |
| |
| if not body: |
| raise Exception('No bench outputs. Most likely there are no matching bench' |
| ' prefix (%s) in Flags -b for platforms %s.\nPlease also ' |
| 'check if the bench graph URLs are valid at %s.' % ( |
| bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE)) |
| if out_file: |
| f = open(out_file, 'w+') |
| f.write(HTML_HEAD + body + HTML_SUFFIX) |
| f.close() |
| else: |
| print HTML_HEAD + body + HTML_SUFFIX |
| |
| |
| if '__main__' == __name__: |
| main() |
