blob: b3c85700ec385d73e3ebf6ed4db012ae37bc5143 [file] [log] [blame]
import os
import sys
from collections import OrderedDict, defaultdict
from mozlog import reader
from mozlog.formatters import JSONFormatter, TbplFormatter
from mozlog.handlers import BaseHandler, LogLevelFilter, StreamHandler
from markdown import markdown_adjust, table
from wptrunner import wptrunner
class LogActionFilter(BaseHandler):
"""Handler that filters out messages not of a given set of actions.
Subclasses BaseHandler.
:param inner: Handler to use for messages that pass this filter
:param actions: List of actions for which to fire the handler
"""
def __init__(self, inner, actions):
"""Extend BaseHandler and set inner and actions props on self."""
BaseHandler.__init__(self, inner)
self.inner = inner
self.actions = actions
def __call__(self, item):
"""Invoke handler if action is in list passed as constructor param."""
if item["action"] in self.actions:
return self.inner(item)
class LogHandler(reader.LogHandler):
"""Handle updating test and subtest status in log.
Subclasses reader.LogHandler.
"""
def __init__(self):
self.results = OrderedDict()
def find_or_create_test(self, data):
test_name = data["test"]
if self.results.get(test_name):
return self.results[test_name]
test = {
"subtests": OrderedDict(),
"status": defaultdict(int)
}
self.results[test_name] = test
return test
def find_or_create_subtest(self, data):
test = self.find_or_create_test(data)
subtest_name = data["subtest"]
if test["subtests"].get(subtest_name):
return test["subtests"][subtest_name]
subtest = {
"status": defaultdict(int),
"messages": set()
}
test["subtests"][subtest_name] = subtest
return subtest
def test_status(self, data):
subtest = self.find_or_create_subtest(data)
subtest["status"][data["status"]] += 1
if data.get("message"):
subtest["messages"].add(data["message"])
def test_end(self, data):
test = self.find_or_create_test(data)
test["status"][data["status"]] += 1
def is_inconsistent(results_dict, iterations):
"""Return whether or not a single test is inconsistent."""
return len(results_dict) > 1 or sum(results_dict.values()) != iterations
def process_results(log, iterations):
"""Process test log and return overall results and list of inconsistent tests."""
inconsistent = []
handler = LogHandler()
reader.handle_log(reader.read(log), handler)
results = handler.results
for test_name, test in results.iteritems():
if is_inconsistent(test["status"], iterations):
inconsistent.append((test_name, None, test["status"], []))
for subtest_name, subtest in test["subtests"].iteritems():
if is_inconsistent(subtest["status"], iterations):
inconsistent.append((test_name, subtest_name, subtest["status"], subtest["messages"]))
return results, inconsistent
def err_string(results_dict, iterations):
"""Create and return string with errors from test run."""
rv = []
total_results = sum(results_dict.values())
for key, value in sorted(results_dict.items()):
rv.append("%s%s" %
(key, ": %s/%s" % (value, iterations) if value != iterations else ""))
if total_results < iterations:
rv.append("MISSING: %s/%s" % (iterations - total_results, iterations))
rv = ", ".join(rv)
if is_inconsistent(results_dict, iterations):
rv = "**%s**" % rv
return rv
def write_inconsistent(log, inconsistent, iterations):
"""Output inconsistent tests to logger.error."""
log("## Unstable results ##\n")
strings = [(
"`%s`" % markdown_adjust(test),
("`%s`" % markdown_adjust(subtest)) if subtest else "",
err_string(results, iterations),
("`%s`" % markdown_adjust(";".join(messages))) if len(messages) else "")
for test, subtest, results, messages in inconsistent]
table(["Test", "Subtest", "Results", "Messages"], strings, log)
def write_results(log, results, iterations, pr_number=None, use_details=False):
log("## All results ##\n")
if use_details:
log("<details>\n")
log("<summary>%i %s ran</summary>\n\n" % (len(results),
"tests" if len(results) > 1
else "test"))
for test_name, test in results.iteritems():
baseurl = "http://w3c-test.org/submissions"
if "https" in os.path.splitext(test_name)[0].split(".")[1:]:
baseurl = "https://w3c-test.org/submissions"
title = test_name
if use_details:
log("<details>\n")
if pr_number:
title = "<a href=\"%s/%s%s\">%s</a>" % (baseurl, pr_number, test_name, title)
log('<summary>%s</summary>\n\n' % title)
else:
log("### %s ###" % title)
strings = [("", err_string(test["status"], iterations), "")]
strings.extend(((
("`%s`" % markdown_adjust(subtest_name)) if subtest else "",
err_string(subtest["status"], iterations),
("`%s`" % markdown_adjust(';'.join(subtest["messages"]))) if len(subtest["messages"]) else "")
for subtest_name, subtest in test["subtests"].items()))
table(["Subtest", "Results", "Messages"], strings, log)
if use_details:
log("</details>\n")
if use_details:
log("</details>\n")
def run(venv, logger, **kwargs):
kwargs["pause_after_test"] = False
if kwargs["repeat"] == 1:
kwargs["repeat"] = 10
handler = LogActionFilter(
LogLevelFilter(
StreamHandler(
sys.stdout,
TbplFormatter()
),
"WARNING"),
["log", "process_output"])
# There is a public API for this in the next mozlog
initial_handlers = logger._state.handlers
logger._state.handlers = []
with open("raw.log", "wb") as log:
# Setup logging for wptrunner that keeps process output and
# warning+ level logs only
logger.add_handler(handler)
logger.add_handler(StreamHandler(log, JSONFormatter()))
wptrunner.run_tests(**kwargs)
logger._state.handlers = initial_handlers
with open("raw.log", "rb") as log:
results, inconsistent = process_results(log, kwargs["repeat"])
return kwargs["repeat"], results, inconsistent