|
#!/usr/bin/python |
|
import json |
|
import optparse |
|
import os |
|
import sys |
|
|
|
from webkitpy.common.host import Host |
|
from webkitpy.layout_tests.port import platform_options, configuration_options |
|
|
|
|
|
def main(argv): |
|
parser = optparse.OptionParser(usage='%prog [path-to-results.json]') |
|
parser.add_option('--failures', action='store_true', |
|
help='show failing tests') |
|
parser.add_option('--flakes', action='store_true', |
|
help='show flaky tests') |
|
parser.add_option('--expected', action='store_true', |
|
help='include expected results along with unexpected') |
|
parser.add_option('--passes', action='store_true', |
|
help='show passing tests') |
|
parser.add_option('--ignored-failures-path', action='store', |
|
help='ignore failures seen in a previous run') |
|
parser.add_options(platform_options()) |
|
parser.add_options(configuration_options()) |
|
options, args = parser.parse_args(argv) |
|
|
|
host = Host() |
|
if args: |
|
if args[0] == '-': |
|
txt = sys.stdin.read() |
|
elif os.path.exists(args[0]): |
|
with open(args[0], 'r') as fp: |
|
txt = fp.read() |
|
else: |
|
print >> sys.stderr, "file not found: %s" % args[0] |
|
sys.exit(1) |
|
else: |
|
txt = host.filesystem.read_text_file(host.filesystem.join(host.port_factory.get(options=options).results_directory(), 'full_results.json')) |
|
|
|
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'): |
|
txt = txt[12:-2] # ignore optional JSONP wrapper |
|
results = json.loads(txt) |
|
|
|
passes, failures, flakes = decode_results(results, options.expected) |
|
|
|
tests_to_print = [] |
|
if options.passes: |
|
tests_to_print += passes.keys() |
|
if options.failures: |
|
tests_to_print += failures.keys() |
|
if options.flakes: |
|
tests_to_print += flakes.keys() |
|
print "\n".join(sorted(tests_to_print)) |
|
|
|
if options.ignored_failures_path: |
|
with open(options.ignored_failures_path, 'r') as fp: |
|
txt = fp.read() |
|
if txt.startswith('ADD_RESULTS(') and txt.endswith(');'): |
|
txt = txt[12:-2] # ignore optional JSONP wrapper |
|
results = json.loads(txt) |
|
_, ignored_failures, _ = decode_results(results, options.expected) |
|
new_failures = set(failures.keys()) - set(ignored_failures.keys()) |
|
if new_failures: |
|
print "New failures:" |
|
print "\n".join(sorted(new_failures)) |
|
print |
|
if ignored_failures: |
|
print "Ignored failures:" |
|
print "\n".join(sorted(ignored_failures.keys())) |
|
if new_failures: |
|
return 1 |
|
return 0 |
|
|
|
|
|
def decode_results(results, include_expected=False): |
|
tests = convert_trie_to_flat_paths(results['tests']) |
|
failures = {} |
|
flakes = {} |
|
passes = {} |
|
for (test, result) in tests.iteritems(): |
|
if include_expected or result.get('is_unexpected'): |
|
actual_results = result['actual'].split() |
|
expected_results = result['expected'].split() |
|
if len(actual_results) > 1: |
|
if actual_results[1] in expected_results: |
|
flakes[test] = actual_results[0] |
|
else: |
|
# We report the first failure type back, even if the second |
|
# was more severe. |
|
failures[test] = actual_results[0] |
|
elif actual_results[0] == 'PASS': |
|
passes[test] = result |
|
else: |
|
failures[test] = actual_results[0] |
|
|
|
return (passes, failures, flakes) |
|
|
|
|
|
def convert_trie_to_flat_paths(trie, prefix=None): |
|
# Cloned from webkitpy.layout_tests.layout_package.json_results_generator |
|
# so that this code can stand alone. |
|
result = {} |
|
for name, data in trie.iteritems(): |
|
if prefix: |
|
name = prefix + "/" + name |
|
|
|
if len(data) and not "actual" in data and not "expected" in data: |
|
result.update(convert_trie_to_flat_paths(data, name)) |
|
else: |
|
result[name] = data |
|
|
|
return result |
|
|
|
|
|
if __name__ == '__main__': |
|
sys.exit(main(sys.argv[1:])) |
