| #!/usr/bin/python |
| # Copyright 2018 the V8 project authors. All rights reserved. |
| |
| ''' |
| python %prog -c <command> [options] |
| |
| Local benchmark runner. |
| The -c option is mandatory. |
| ''' |
| |
| # for py2/py3 compatibility |
| from __future__ import print_function |
| |
| import math |
| from optparse import OptionParser |
| import os |
| import re |
| import subprocess |
| import sys |
| import time |
| |
| def GeometricMean(numbers): |
| log = sum([math.log(n) for n in numbers]) |
| return math.pow(math.e, log / len(numbers)) |
| |
| |
| class BenchmarkSuite(object): |
| |
| def __init__(self, name): |
| self.name = name |
| self.results = {} |
| self.tests = [] |
| self.avgresult = {} |
| self.sigmaresult = {} |
| self.numresult = {} |
| self.kClassicScoreSuites = ["SunSpider", "Kraken"] |
| self.kGeometricScoreSuites = ["Octane"] |
| |
| |
| def RecordResult(self, test, result): |
| if test not in self.tests: |
| self.tests += [test] |
| self.results[test] = [] |
| self.results[test] += [int(result)] |
| |
| def ThrowAwayWorstResult(self, results): |
| if len(results) <= 1: return |
| if self.name in self.kClassicScoreSuites: |
| results.pop() |
| elif self.name in self.kGeometricScoreSuites: |
| del results[0] |
| |
| def ProcessResults(self, opts): |
| for test in self.tests: |
| results = self.results[test] |
| results.sort() |
| self.ThrowAwayWorstResult(results) |
| mean = sum(results) * 1.0 / len(results) |
| self.avgresult[test] = mean |
| sigma_divisor = len(results) - 1 |
| if sigma_divisor == 0: |
| sigma_divisor = 1 |
| self.sigmaresult[test] = math.sqrt( |
| sum((x - mean) ** 2 for x in results) / sigma_divisor) |
| self.numresult[test] = len(results) |
| if opts.verbose: |
| if not test in ["Octane"]: |
| print("%s,%.1f,%.2f,%d" % |
| (test, self.avgresult[test], |
| self.sigmaresult[test], self.numresult[test])) |
| |
| def ComputeScoreGeneric(self): |
| self.score = 0 |
| self.sigma = 0 |
| for test in self.tests: |
| self.score += self.avgresult[test] |
| self.sigma += self.sigmaresult[test] |
| self.num = self.numresult[test] |
| |
| def ComputeScoreV8Octane(self, name): |
| # The score for the run is stored with the form |
| # "Octane-octane2.1(Score): <score>" |
| found_name = '' |
| for s in self.avgresult.keys(): |
| if re.search("^Octane", s): |
| found_name = s |
| break |
| |
| self.score = self.avgresult[found_name] |
| self.sigma = 0 |
| for test in self.tests: |
| self.sigma += self.sigmaresult[test] |
| self.num = self.numresult[test] |
| self.sigma /= len(self.tests) |
| |
| def ComputeScore(self): |
| if self.name in self.kClassicScoreSuites: |
| self.ComputeScoreGeneric() |
| elif self.name in self.kGeometricScoreSuites: |
| self.ComputeScoreV8Octane(self.name) |
| else: |
| print("Don't know how to compute score for suite: '%s'" % self.name) |
| |
| def IsBetterThan(self, other): |
| if self.name in self.kClassicScoreSuites: |
| return self.score < other.score |
| elif self.name in self.kGeometricScoreSuites: |
| return self.score > other.score |
| else: |
| print("Don't know how to compare score for suite: '%s'" % self.name) |
| |
| |
| class BenchmarkRunner(object): |
| def __init__(self, args, current_directory, opts): |
| self.best = {} |
| self.second_best = {} |
| self.args = args |
| self.opts = opts |
| self.current_directory = current_directory |
| self.outdir = os.path.join(opts.cachedir, "_benchmark_runner_data") |
| |
| def Run(self): |
| if not os.path.exists(self.outdir): |
| os.mkdir(self.outdir) |
| |
| self.RunCommand() |
| # Figure out the suite from the command line (heuristic) or the current |
| # working directory. |
| teststr = opts.command.lower() + " " + self.current_directory.lower() |
| if teststr.find('octane') >= 0: |
| suite = 'Octane' |
| elif teststr.find('sunspider') >= 0: |
| suite = 'SunSpider' |
| elif teststr.find('kraken') >= 0: |
| suite = 'Kraken' |
| else: |
| suite = 'Generic' |
| |
| self.ProcessOutput(suite) |
| |
| def RunCommand(self): |
| for i in range(self.opts.runs): |
| outfile = "%s/out.%d.txt" % (self.outdir, i) |
| if os.path.exists(outfile) and not self.opts.force: |
| continue |
| print("run #%d" % i) |
| cmdline = "%s > %s" % (self.opts.command, outfile) |
| subprocess.call(cmdline, shell=True) |
| time.sleep(self.opts.sleep) |
| |
| def ProcessLine(self, line): |
| # Octane puts this line in before score. |
| if line == "----": |
| return (None, None) |
| |
| # Kraken or Sunspider? |
| g = re.match("(?P<test_name>\w+(-\w+)*)\(RunTime\): (?P<score>\d+) ms\.", \ |
| line) |
| if g == None: |
| # Octane? |
| g = re.match("(?P<test_name>\w+): (?P<score>\d+)", line) |
| if g == None: |
| g = re.match("Score \(version [0-9]+\): (?P<score>\d+)", line) |
| if g != None: |
| return ('Octane', g.group('score')) |
| else: |
| # Generic? |
| g = re.match("(?P<test_name>\w+)\W+(?P<score>\d+)", line) |
| if g == None: |
| return (None, None) |
| return (g.group('test_name'), g.group('score')) |
| |
| def ProcessOutput(self, suitename): |
| suite = BenchmarkSuite(suitename) |
| for i in range(self.opts.runs): |
| outfile = "%s/out.%d.txt" % (self.outdir, i) |
| with open(outfile, 'r') as f: |
| for line in f: |
| (test, result) = self.ProcessLine(line) |
| if test != None: |
| suite.RecordResult(test, result) |
| |
| suite.ProcessResults(self.opts) |
| suite.ComputeScore() |
| print(("%s,%.1f,%.2f,%d " % |
| (suite.name, suite.score, suite.sigma, suite.num)), end='') |
| if self.opts.verbose: |
| print("") |
| print("") |
| |
| |
| if __name__ == '__main__': |
| parser = OptionParser(usage=__doc__) |
| parser.add_option("-c", "--command", dest="command", |
| help="Command to run the test suite.") |
| parser.add_option("-r", "--runs", dest="runs", default=4, |
| help="Number of runs") |
| parser.add_option("-v", "--verbose", dest="verbose", action="store_true", |
| default=False, help="Print results for each test") |
| parser.add_option("-f", "--force", dest="force", action="store_true", |
| default=False, |
| help="Force re-run even if output files exist") |
| parser.add_option("-z", "--sleep", dest="sleep", default=0, |
| help="Number of seconds to sleep between runs") |
| parser.add_option("-d", "--run-directory", dest="cachedir", |
| help="Directory where a cache directory will be created") |
| (opts, args) = parser.parse_args() |
| opts.runs = int(opts.runs) |
| opts.sleep = int(opts.sleep) |
| |
| if not opts.command: |
| print("You must specify the command to run (-c). Aborting.") |
| sys.exit(1) |
| |
| cachedir = os.path.abspath(os.getcwd()) |
| if not opts.cachedir: |
| opts.cachedir = cachedir |
| if not os.path.exists(opts.cachedir): |
| print("Directory " + opts.cachedir + " is not valid. Aborting.") |
| sys.exit(1) |
| |
| br = BenchmarkRunner(args, os.getcwd(), opts) |
| br.Run() |
