| from __future__ import print_function | |
| import argparse | |
| import json | |
| import sys | |
| from cgi import escape | |
| from collections import defaultdict | |
| import types | |
| def html_escape(item, escape_quote=False): | |
| if isinstance(item, types.StringTypes): | |
| rv = escape(item) | |
| if escape_quote: | |
| rv = rv.replace('"', """) | |
| return rv | |
| else: | |
| return item | |
| class Raw(object): | |
| """Simple wrapper around a string to stop it being escaped by html_escape""" | |
| def __init__(self, value): | |
| self.value = value | |
| def __unicode__(self): | |
| return unicode(self.value) | |
| class Node(object): | |
| """Node structure used when building HTML""" | |
| def __init__(self, name, attrs, children): | |
| #Need list of void elements | |
| self.name = name | |
| self.attrs = attrs | |
| self.children = children | |
| def __unicode__(self): | |
| if self.attrs: | |
| #Need to escape | |
| attrs_unicode = " " + " ".join("%s=\"%s\"" % (html_escape(key), | |
| html_escape(value, | |
| escape_quote=True)) | |
| for key, value in self.attrs.items()) | |
| else: | |
| attrs_unicode = "" | |
| return "<%s%s>%s</%s>\n" % (self.name, | |
| attrs_unicode, | |
| "".join(unicode(html_escape(item)) | |
| for item in self.children), | |
| self.name) | |
| def __str__(self): | |
| return unicode(self).encode("utf8") | |
| class RootNode(object): | |
| """Special Node representing the document root""" | |
| def __init__(self, *children): | |
| self.children = ["<!DOCTYPE html>"] + list(children) | |
| def __unicode__(self): | |
| return "".join(unicode(item) for item in self.children) | |
| def __str__(self): | |
| return unicode(self).encode("utf8") | |
| def flatten(iterable): | |
| """Flatten a list of lists by one level so that | |
| [1,["abc"], "def",[2, [3]]] | |
| becomes | |
| [1, "abc", "def", 2, [3]]""" | |
| rv = [] | |
| for item in iterable: | |
| if hasattr(item, "__iter__") and not isinstance(item, types.StringTypes): | |
| rv.extend(item) | |
| else: | |
| rv.append(item) | |
| return rv | |
| class HTML(object): | |
| """Simple HTML templating system. An instance of this class can create | |
| element nodes by calling methods with the same name as the element, | |
| passing in children as positional arguments or as a list, and attributes | |
| as keyword arguments, with _ replacing - and trailing _ for python keywords | |
| e.g. | |
| h = HTML() | |
| print(h.html( | |
| html.head(), | |
| html.body([html.h1("Hello World!")], class_="body-class") | |
| )) | |
| Would give | |
| <!DOCTYPE html><html><head></head><body class="body-class"><h1>Hello World!</h1></body></html>""" | |
| def __getattr__(self, name): | |
| def make_html(self, *content, **attrs): | |
| for attr_name in attrs.keys(): | |
| if "_" in attr_name: | |
| new_name = attr_name.replace("_", "-") | |
| if new_name.endswith("-"): | |
| new_name = new_name[:-1] | |
| attrs[new_name] = attrs.pop(attr_name) | |
| return Node(name, attrs, flatten(content)) | |
| method = types.MethodType(make_html, self, HTML) | |
| setattr(self, name, method) | |
| return method | |
| def __call__(self, *children): | |
| return RootNode(*flatten(children)) | |
| h = HTML() | |
| class TestResult(object): | |
| """Simple holder for the results of a single test in a single UA""" | |
| def __init__(self, test): | |
| self.test = test | |
| self.results = {} | |
| def __cmp__(self, other): | |
| return self.test == other.test | |
| def __hash__(self): | |
| return hash(self.test) | |
| def load_data(args): | |
| """Load data treating args as a list of UA name, filename pairs""" | |
| pairs = [] | |
| for i in xrange(0, len(args), 2): | |
| pairs.append(args[i:i+2]) | |
| rv = {} | |
| for UA, filename in pairs: | |
| with open(filename) as f: | |
| rv[UA] = json.load(f) | |
| return rv | |
| def test_id(id): | |
| """Convert a test id in JSON into an immutable object that | |
| can be used as a dictionary key""" | |
| if isinstance(id, list): | |
| return tuple(id) | |
| else: | |
| return id | |
| def all_tests(data): | |
| tests = defaultdict(set) | |
| for UA, results in data.items(): | |
| for result in results["results"]: | |
| id = test_id(result["test"]) | |
| tests[id] |= set(subtest["name"] for subtest in result["subtests"]) | |
| return tests | |
| def group_results(data): | |
| """Produce a list of UAs and a dictionary mapping specific tests to their | |
| status in all UAs e.g. | |
| ["UA1", "UA2"], {"test_id":{"harness":{"UA1": (status1, message1), | |
| "UA2": (status2, message2)}, | |
| "subtests":{"subtest1": "UA1": (status1-1, message1-1), | |
| "UA2": (status2-1, message2-1)}}} | |
| Status and message are None if the test didn't run in a particular UA. | |
| Message is None if the test didn't produce a message""" | |
| tests = all_tests(data) | |
| UAs = data.keys() | |
| def result(): | |
| return { | |
| "harness": dict((UA, (None, None)) for UA in UAs), | |
| "subtests": None # init this later | |
| } | |
| results_by_test = defaultdict(result) | |
| for UA, results in data.items(): | |
| for test_data in results["results"]: | |
| id = test_id(test_data["test"]) | |
| result = results_by_test[id] | |
| if result["subtests"] is None: | |
| result["subtests"] = dict( | |
| (name, dict((UA, (None, None)) for UA in UAs)) for name in tests[id] | |
| ) | |
| result["harness"][UA] = (test_data["status"], test_data["message"]) | |
| for subtest in test_data["subtests"]: | |
| result["subtests"][subtest["name"]][UA] = (subtest["status"], | |
| subtest["message"]) | |
| return UAs, results_by_test | |
| def status_cell(status, message=None): | |
| """Produce a table cell showing the status of a test""" | |
| status = status if status is not None else "NONE" | |
| kwargs = {} | |
| if message: | |
| kwargs["title"] = message | |
| status_text = status.title() | |
| return h.td(status_text, class_="status " + status, | |
| **kwargs) | |
| def test_link(test_id, subtest=None): | |
| """Produce an <a> element linking to a test""" | |
| if isinstance(test_id, types.StringTypes): | |
| rv = [h.a(test_id, href=test_id)] | |
| else: | |
| rv = [h.a(test_id[0], href=test_id[0]), | |
| " %s " % test_id[1], | |
| h.a(test_id[2], href=test_id[2])] | |
| if subtest is not None: | |
| rv.append(" [%s]" % subtest) | |
| return rv | |
| def summary(UAs, results_by_test): | |
| """Render the implementation report summary""" | |
| not_passing = [] | |
| for test, results in results_by_test.items(): | |
| if not any(item[0] in ("PASS", "OK") for item in results["harness"].values()): | |
| not_passing.append((test, None)) | |
| for subtest_name, subtest_results in results["subtests"].items(): | |
| if not any(item[0] == "PASS" for item in subtest_results.values()): | |
| not_passing.append((test, subtest_name)) | |
| if not_passing: | |
| rv = [ | |
| h.p("The following tests failed to pass in all UAs:"), | |
| h.ul([h.li(test_link(test, subtest)) | |
| for test, subtest in not_passing]) | |
| ] | |
| else: | |
| rv = "All tests passed in at least one UA" | |
| return rv | |
| def result_rows(UAs, test, result): | |
| """Render the results for each test run""" | |
| yield h.tr( | |
| h.td( | |
| test_link(test), | |
| rowspan=(1 + len(result["subtests"])) | |
| ), | |
| h.td(), | |
| [status_cell(status, message) | |
| for UA, (status, message) in sorted(result["harness"].items())], | |
| class_="test" | |
| ) | |
| for name, subtest_result in sorted(result["subtests"].items()): | |
| yield h.tr( | |
| h.td(name), | |
| [status_cell(status, message) | |
| for UA, (status, message) in sorted(subtest_result.items())], | |
| class_="subtest" | |
| ) | |
| def result_bodies(UAs, results_by_test): | |
| return [h.tbody(result_rows(UAs, test, result)) | |
| for test, result in sorted(results_by_test.items())] | |
| def generate_html(UAs, results_by_test): | |
| """Generate all the HTML output""" | |
| return h(h.html( | |
| h.head( | |
| h.meta(charset="utf8"), | |
| h.title("Implementation Report"), | |
| h.link(href="report.css", rel="stylesheet")), | |
| h.body( | |
| h.h1("Implementation Report"), | |
| h.h2("Summary"), | |
| summary(UAs, results_by_test), | |
| h.h2("Full Results"), | |
| h.table( | |
| h.thead( | |
| h.tr( | |
| h.th("Test"), | |
| h.th("Subtest"), | |
| [h.th(UA) for UA in sorted(UAs)])), | |
| result_bodies(UAs, results_by_test))))) | |
| def main(filenames): | |
| data = load_data(filenames) | |
| UAs, results_by_test = group_results(data) | |
| return generate_html(UAs, results_by_test) | |
| if __name__ == "__main__": | |
| if not sys.argv[1:]: | |
| print("""Please supply a list of UA name, filename pairs e.g. | |
| python report.py Firefox firefox.json Chrome chrome.json IE internet_explorer.json""") | |
| print(main(sys.argv[1:])) |