| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import cPickle |
| |
| from webkitpy.layout_tests.models import test_expectations |
| |
| |
| def is_reftest_failure(failure_list): |
| failure_types = [type(f) for f in failure_list] |
| return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection( |
| failure_types) |
| |
| # FIXME: This is backwards. Each TestFailure subclass should know what |
| # test_expectation type it corresponds too. Then this method just |
| # collects them all from the failure list and returns the worst one. |
| |
| |
| def determine_result_type(failure_list): |
| """Takes a set of test_failures and returns which result type best fits |
| the list of failures. "Best fits" means we use the worst type of failure. |
| |
| Returns: |
| one of the test_expectations result types - PASS, FAIL, CRASH, etc. |
| """ |
| |
| if not failure_list or len(failure_list) == 0: |
| return test_expectations.PASS |
| |
| failure_types = [type(f) for f in failure_list] |
| if FailureCrash in failure_types: |
| return test_expectations.CRASH |
| elif FailureLeak in failure_types: |
| return test_expectations.LEAK |
| elif FailureTimeout in failure_types: |
| return test_expectations.TIMEOUT |
| elif FailureEarlyExit in failure_types: |
| return test_expectations.SKIP |
| elif (FailureMissingResult in failure_types or |
| FailureMissingImage in failure_types or |
| FailureMissingImageHash in failure_types or |
| FailureMissingAudio in failure_types): |
| return test_expectations.MISSING |
| else: |
| is_text_failure = (FailureTextMismatch in failure_types or |
| FailureTestHarnessAssertion in failure_types) |
| is_image_failure = (FailureImageHashIncorrect in failure_types or |
| FailureImageHashMismatch in failure_types or |
| is_reftest_failure(failure_list)) |
| is_audio_failure = (FailureAudioMismatch in failure_types) |
| if is_text_failure and is_image_failure: |
| return test_expectations.IMAGE_PLUS_TEXT |
| elif is_text_failure: |
| return test_expectations.TEXT |
| elif is_image_failure: |
| return test_expectations.IMAGE |
| elif is_audio_failure: |
| return test_expectations.AUDIO |
| else: |
| raise ValueError('unclassifiable set of failures: ' |
| + str(failure_types)) |
| |
| |
| class TestFailure(object): |
| """Abstract base class that defines the failure interface.""" |
| |
| @staticmethod |
| def loads(s): |
| """Creates a TestFailure object from the specified string.""" |
| return cPickle.loads(s) |
| |
| def message(self): |
| """Returns a string describing the failure in more detail.""" |
| raise NotImplementedError |
| |
| def __eq__(self, other): |
| return self.__class__.__name__ == other.__class__.__name__ |
| |
| def __ne__(self, other): |
| return self.__class__.__name__ != other.__class__.__name__ |
| |
| def __hash__(self): |
| return hash(self.__class__.__name__) |
| |
| def dumps(self): |
| """Returns the string/JSON representation of a TestFailure.""" |
| return cPickle.dumps(self) |
| |
| def driver_needs_restart(self): |
| """Returns True if we should kill the driver before the next test.""" |
| return False |
| |
| |
| class FailureTimeout(TestFailure): |
| |
| def __init__(self, is_reftest=False): |
| super(FailureTimeout, self).__init__() |
| self.is_reftest = is_reftest |
| |
| def message(self): |
| return 'test timed out' |
| |
| def driver_needs_restart(self): |
| return True |
| |
| |
| class FailureCrash(TestFailure): |
| |
| def __init__(self, is_reftest=False, process_name='content_shell', pid=None, has_log=False): |
| super(FailureCrash, self).__init__() |
| self.process_name = process_name |
| self.pid = pid |
| self.is_reftest = is_reftest |
| self.has_log = has_log |
| |
| def message(self): |
| if self.pid: |
| return '%s crashed [pid=%d]' % (self.process_name, self.pid) |
| return self.process_name + ' crashed' |
| |
| def driver_needs_restart(self): |
| return True |
| |
| |
| class FailureLeak(TestFailure): |
| |
| def __init__(self, is_reftest=False, log=''): |
| super(FailureLeak, self).__init__() |
| self.is_reftest = is_reftest |
| self.log = log |
| |
| def message(self): |
| return 'leak detected: %s' % (self.log) |
| |
| |
| class FailureMissingResult(TestFailure): |
| |
| def message(self): |
| return '-expected.txt was missing' |
| |
| |
| class FailureTestHarnessAssertion(TestFailure): |
| |
| def message(self): |
| return 'asserts failed' |
| |
| |
| class FailureTextMismatch(TestFailure): |
| |
| def message(self): |
| return 'text diff' |
| |
| |
| class FailureMissingImageHash(TestFailure): |
| |
| def message(self): |
| return '-expected.png was missing an embedded checksum' |
| |
| |
| class FailureMissingImage(TestFailure): |
| |
| def message(self): |
| return '-expected.png was missing' |
| |
| |
| class FailureImageHashMismatch(TestFailure): |
| |
| def message(self): |
| return 'image diff' |
| |
| |
| class FailureImageHashIncorrect(TestFailure): |
| |
| def message(self): |
| return '-expected.png embedded checksum is incorrect' |
| |
| |
| class FailureReftestMismatch(TestFailure): |
| |
| def __init__(self, reference_filename=None): |
| super(FailureReftestMismatch, self).__init__() |
| self.reference_filename = reference_filename |
| |
| def message(self): |
| return 'reference mismatch' |
| |
| |
| class FailureReftestMismatchDidNotOccur(TestFailure): |
| |
| def __init__(self, reference_filename=None): |
| super(FailureReftestMismatchDidNotOccur, self).__init__() |
| self.reference_filename = reference_filename |
| |
| def message(self): |
| return "reference mismatch didn't happen" |
| |
| |
| class FailureReftestNoImagesGenerated(TestFailure): |
| |
| def __init__(self, reference_filename=None): |
| super(FailureReftestNoImagesGenerated, self).__init__() |
| self.reference_filename = reference_filename |
| |
| def message(self): |
| return "reference didn't generate pixel results." |
| |
| |
| class FailureMissingAudio(TestFailure): |
| |
| def message(self): |
| return 'expected audio result was missing' |
| |
| |
| class FailureAudioMismatch(TestFailure): |
| |
| def message(self): |
| return 'audio mismatch' |
| |
| |
| class FailureEarlyExit(TestFailure): |
| |
| def message(self): |
| return 'skipped due to early exit' |
| |
| |
| # Convenient collection of all failure classes for anything that might |
| # need to enumerate over them all. |
| ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, |
| FailureTestHarnessAssertion, |
| FailureTextMismatch, FailureMissingImageHash, |
| FailureMissingImage, FailureImageHashMismatch, |
| FailureImageHashIncorrect, FailureReftestMismatch, |
| FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated, |
| FailureMissingAudio, FailureAudioMismatch, |
| FailureEarlyExit) |