| #!/usr/bin/env python |
| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Main functions for the Layout Test Analyzer module.""" |
| |
| from datetime import datetime |
| import optparse |
| import os |
| import sys |
| import time |
| |
| import layouttest_analyzer_helpers |
| from layouttest_analyzer_helpers import DEFAULT_REVISION_VIEW_URL |
| import layouttests |
| from layouttests import DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION |
| |
| from test_expectations import TestExpectations |
| from trend_graph import TrendGraph |
| |
| # Predefined result directory. |
| DEFAULT_RESULT_DIR = 'result' |
| # TODO(shadi): Remove graph functions as they are not used any more. |
| DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html') |
| # TODO(shadi): Check if these files are needed any more. |
| DEFAULT_STATS_CSV_FILENAME = 'stats.csv' |
| DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv' |
| # TODO(shadi): These are used only for |debug| mode. What is debug mode for? |
| # AFAIK, we don't run debug mode, should be safe to remove. |
| # Predefined result files for debug. |
| CUR_TIME_FOR_DEBUG = '2011-09-11-19' |
| CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR, |
| CUR_TIME_FOR_DEBUG) |
| PREV_TIME_FOR_DEBUG = '2011-09-11-18' |
| |
| # Text to append at the end of every analyzer result email. |
| DEFAULT_EMAIL_APPEND_TEXT = ( |
| '<b><a href="https://groups.google.com/a/google.com/group/' |
| 'layout-test-analyzer-result/topics">Email History</a></b><br>' |
| ) |
| |
| |
| def ParseOption(): |
| """Parse command-line options using OptionParser. |
| |
| Returns: |
| an object containing all command-line option information. |
| """ |
| option_parser = optparse.OptionParser() |
| |
| option_parser.add_option('-r', '--receiver-email-address', |
| dest='receiver_email_address', |
| help=('receiver\'s email address. ' |
| 'Result email is not sent if this is not ' |
| 'specified.')) |
| option_parser.add_option('-g', '--debug-mode', dest='debug', |
| help=('Debug mode is used when you want to debug ' |
| 'the analyzer by using local file rather ' |
| 'than getting data from SVN. This shortens ' |
| 'the debugging time (off by default).'), |
| action='store_true', default=False) |
| option_parser.add_option('-t', '--trend-graph-location', |
| dest='trend_graph_location', |
| help=('Location of the bug trend file; ' |
| 'file is expected to be in Google ' |
| 'Visualization API trend-line format ' |
| '(defaults to %default).'), |
| default=DEFAULT_GRAPH_FILE) |
| option_parser.add_option('-n', '--test-group-file-location', |
| dest='test_group_file_location', |
| help=('Location of the test group file; ' |
| 'file is expected to be in CSV format ' |
| 'and lists all test name patterns. ' |
| 'When this option is not specified, ' |
| 'the value of --test-group-name is used ' |
| 'for a test name pattern.'), |
| default=None) |
| option_parser.add_option('-x', '--test-group-name', |
| dest='test_group_name', |
| help=('A name of test group. Either ' |
| '--test_group_file_location or this option ' |
| 'needs to be specified.')) |
| option_parser.add_option('-d', '--result-directory-location', |
| dest='result_directory_location', |
| help=('Name of result directory location ' |
| '(default to %default).'), |
| default=DEFAULT_RESULT_DIR) |
| option_parser.add_option('-b', '--email-appended-text-file-location', |
| dest='email_appended_text_file_location', |
| help=('File location of the email appended text. ' |
| 'The text is appended in the status email. ' |
| '(default to %default and no text is ' |
| 'appended in that case).'), |
| default=None) |
| option_parser.add_option('-c', '--email-only-change-mode', |
| dest='email_only_change_mode', |
| help=('With this mode, email is sent out ' |
| 'only when there is a change in the ' |
| 'analyzer result compared to the previous ' |
| 'result (off by default)'), |
| action='store_true', default=False) |
| option_parser.add_option('-q', '--dashboard-file-location', |
| dest='dashboard_file_location', |
| help=('Location of dashboard file. The results are ' |
| 'not reported to the dashboard if this ' |
| 'option is not specified.')) |
| option_parser.add_option('-z', '--issue-detail-mode', |
| dest='issue_detail_mode', |
| help=('With this mode, email includes issue details ' |
| '(links to the flakiness dashboard)' |
| ' (off by default)'), |
| action='store_true', default=False) |
| return option_parser.parse_args()[0] |
| |
| |
| def GetCurrentAndPreviousResults(debug, test_group_file_location, |
| test_group_name, result_directory_location): |
| """Get current and the latest previous analyzer results. |
| |
| In debug mode, they are read from predefined files. In non-debug mode, |
| current analyzer results are dynamically obtained from Blink SVN and |
| the latest previous result is read from the corresponding file. |
| |
| Args: |
| debug: please refer to |options|. |
| test_group_file_location: please refer to |options|. |
| test_group_name: please refer to |options|. |
| result_directory_location: please refer to |options|. |
| |
| Returns: |
| a tuple of the following: |
| prev_time: the previous time string that is compared against. |
| prev_analyzer_result_map: previous analyzer result map. Please refer to |
| layouttest_analyzer_helpers.AnalyzerResultMap. |
| analyzer_result_map: current analyzer result map. Please refer to |
| layouttest_analyzer_helpers.AnalyzerResultMap. |
| """ |
| if not debug: |
| if not test_group_file_location and not test_group_name: |
| print ('Either --test-group-name or --test_group_file_location must be ' |
| 'specified. Exiting this program.') |
| sys.exit() |
| filter_names = [] |
| if test_group_file_location and os.path.exists(test_group_file_location): |
| filter_names = layouttests.LayoutTests.GetLayoutTestNamesFromCSV( |
| test_group_file_location) |
| parent_location_list = ( |
| layouttests.LayoutTests.GetParentDirectoryList(filter_names)) |
| recursion = True |
| else: |
| # When test group CSV file is not specified, test group name |
| # (e.g., 'media') is used for getting layout tests. |
| # The tests are in |
| # http://src.chromium.org/blink/trunk/LayoutTests/media |
| # Filtering is not set so all HTML files are considered as valid tests. |
| # Also, we look for the tests recursively. |
| if not test_group_file_location or ( |
| not os.path.exists(test_group_file_location)): |
| print ('Warning: CSV file (%s) does not exist. So it is ignored and ' |
| '%s is used for obtaining test names') % ( |
| test_group_file_location, test_group_name) |
| if not test_group_name.endswith('/'): |
| test_group_name += '/' |
| parent_location_list = [test_group_name] |
| filter_names = None |
| recursion = True |
| layouttests_object = layouttests.LayoutTests( |
| parent_location_list=parent_location_list, recursion=recursion, |
| filter_names=filter_names) |
| analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap( |
| layouttests_object.JoinWithTestExpectation(TestExpectations())) |
| result = layouttest_analyzer_helpers.FindLatestResult( |
| result_directory_location) |
| if result: |
| (prev_time, prev_analyzer_result_map) = result |
| else: |
| prev_time = None |
| prev_analyzer_result_map = None |
| else: |
| analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap.Load( |
| CURRENT_RESULT_FILE_FOR_DEBUG) |
| prev_time = PREV_TIME_FOR_DEBUG |
| prev_analyzer_result_map = ( |
| layouttest_analyzer_helpers.AnalyzerResultMap.Load( |
| os.path.join(DEFAULT_RESULT_DIR, prev_time))) |
| return (prev_time, prev_analyzer_result_map, analyzer_result_map) |
| |
| |
| def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, |
| appended_text_to_email, email_only_change_mode, debug, |
| receiver_email_address, test_group_name, issue_detail_mode): |
| """Send result status email. |
| |
| Args: |
| prev_time: the previous time string that is compared against. |
| prev_analyzer_result_map: previous analyzer result map. Please refer to |
| layouttest_analyzer_helpers.AnalyzerResultMap. |
| analyzer_result_map: current analyzer result map. Please refer to |
| layouttest_analyzer_helpers.AnalyzerResultMap. |
| appended_text_to_email: the text string to append to the status email. |
| email_only_change_mode: please refer to |options|. |
| debug: please refer to |options|. |
| receiver_email_address: please refer to |options|. |
| test_group_name: please refer to |options|. |
| issue_detail_mode: please refer to |options|. |
| |
| Returns: |
| a tuple of the following: |
| result_change: a boolean indicating whether there is a change in the |
| result compared with the latest past result. |
| diff_map: please refer to |
| layouttest_analyzer_helpers.SendStatusEmail(). |
| simple_rev_str: a simple version of revision string that is sent in |
| the email. |
| rev: the latest revision number for the given test group. |
| rev_date: the latest revision date for the given test group. |
| email_content: email content string (without |
| |appended_text_to_email|) that will be shown on the dashboard. |
| """ |
| rev = '' |
| rev_date = '' |
| email_content = '' |
| if prev_analyzer_result_map: |
| diff_map = analyzer_result_map.CompareToOtherResultMap( |
| prev_analyzer_result_map) |
| result_change = (any(diff_map['whole']) or any(diff_map['skip']) or |
| any(diff_map['nonskip'])) |
| # Email only when |email_only_change_mode| is False or there |
| # is a change in the result compared to the last result. |
| simple_rev_str = '' |
| if not email_only_change_mode or result_change: |
| prev_time_in_float = datetime.strptime(prev_time, '%Y-%m-%d-%H') |
| prev_time_in_float = time.mktime(prev_time_in_float.timetuple()) |
| if debug: |
| cur_time_in_float = datetime.strptime(CUR_TIME_FOR_DEBUG, |
| '%Y-%m-%d-%H') |
| cur_time_in_float = time.mktime(cur_time_in_float.timetuple()) |
| else: |
| cur_time_in_float = time.time() |
| (rev_str, simple_rev_str, rev, rev_date) = ( |
| layouttest_analyzer_helpers.GetRevisionString(prev_time_in_float, |
| cur_time_in_float, |
| diff_map)) |
| email_content = analyzer_result_map.ConvertToString(prev_time, |
| diff_map, |
| issue_detail_mode) |
| if receiver_email_address: |
| layouttest_analyzer_helpers.SendStatusEmail( |
| prev_time, analyzer_result_map, diff_map, |
| receiver_email_address, test_group_name, |
| appended_text_to_email, email_content, rev_str, |
| email_only_change_mode) |
| if simple_rev_str: |
| simple_rev_str = '\'' + simple_rev_str + '\'' |
| else: |
| simple_rev_str = 'undefined' # GViz uses undefined for NONE. |
| else: |
| # Initial result should be written to tread-graph if there are no previous |
| # results. |
| result_change = True |
| diff_map = None |
| simple_rev_str = 'undefined' |
| email_content = analyzer_result_map.ConvertToString(None, diff_map, |
| issue_detail_mode) |
| return (result_change, diff_map, simple_rev_str, rev, rev_date, |
| email_content) |
| |
| |
| def UpdateTrendGraph(start_time, analyzer_result_map, diff_map, simple_rev_str, |
| trend_graph_location): |
| """Update trend graph in GViz. |
| |
| Annotate the graph with revision information. |
| |
| Args: |
| start_time: the script starting time as a float value. |
| analyzer_result_map: current analyzer result map. Please refer to |
| layouttest_analyzer_helpers.AnalyzerResultMap. |
| diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys. |
| Please refer to |diff_map| in |
| |layouttest_analyzer_helpers.SendStatusEmail()|. |
| simple_rev_str: a simple version of revision string that is sent in |
| the email. |
| trend_graph_location: the location of the trend graph that needs to be |
| updated. |
| |
| Returns: |
| a dictionary that maps result data category ('whole', 'skip', 'nonskip', |
| 'passingrate') to information tuple (a dictionary that maps test name |
| to its description, annotation, simple_rev_string) of the given result |
| data category. These tuples are used for trend graph update. |
| """ |
| # Trend graph update (if specified in the command-line argument) when |
| # there is change from the last result. |
| # Currently, there are two graphs (graph1 is for 'whole', 'skip', |
| # 'nonskip' and the graph2 is for 'passingrate'). Please refer to |
| # graph/graph.html. |
| # Sample JS annotation for graph1: |
| # [new Date(2011,8,12,10,41,32),224,undefined,'',52,undefined, |
| # undefined, 12, 'test1,','<a href="http://t</a>,',], |
| # This example lists 'whole' triple and 'skip' triple and |
| # 'nonskip' triple. Each triple is (the number of tests that belong to |
| # the test group, linked text, a link). The following code generates this |
| # automatically based on rev_string etc. |
| trend_graph = TrendGraph(trend_graph_location) |
| datetime_string = start_time.strftime('%Y,%m,%d,%H,%M,%S') |
| data_map = {} |
| passingrate_anno = '' |
| for test_group in ['whole', 'skip', 'nonskip']: |
| anno = 'undefined' |
| # Extract test description. |
| test_map = {} |
| for (test_name, value) in ( |
| analyzer_result_map.result_map[test_group].iteritems()): |
| test_map[test_name] = value['desc'] |
| test_str = '' |
| links = '' |
| if diff_map and diff_map[test_group]: |
| for i in [0, 1]: |
| for (name, _) in diff_map[test_group][i]: |
| test_str += name + ',' |
| # This is link to test HTML in SVN. |
| links += ('<a href="%s%s">%s</a>' % |
| (DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION, name, name)) |
| if test_str: |
| anno = '\'' + test_str + '\'' |
| # The annotation of passing rate is a union of all annotations. |
| passingrate_anno += anno |
| if links: |
| links = '\'' + links + '\'' |
| else: |
| links = 'undefined' |
| if test_group is 'whole': |
| data_map[test_group] = (test_map, anno, links) |
| else: |
| data_map[test_group] = (test_map, anno, simple_rev_str) |
| if not passingrate_anno: |
| passingrate_anno = 'undefined' |
| data_map['passingrate'] = ( |
| str(analyzer_result_map.GetPassingRate()), passingrate_anno, |
| simple_rev_str) |
| trend_graph.Update(datetime_string, data_map) |
| return data_map |
| |
| |
| def UpdateDashboard(dashboard_file_location, test_group_name, data_map, |
| layouttest_root_path, rev, rev_date, email, |
| email_content): |
| """Update dashboard HTML file. |
| |
| Args: |
| dashboard_file_location: the file location for the dashboard file. |
| test_group_name: please refer to |options|. |
| data_map: a dictionary that maps result data category ('whole', 'skip', |
| 'nonskip', 'passingrate') to information tuple (a dictionary that maps |
| test name to its description, annotation, simple_rev_string) of the |
| given result data category. These tuples are used for trend graph |
| update. |
| layouttest_root_path: A location string where layout tests are stored. |
| rev: the latest revision number for the given test group. |
| rev_date: the latest revision date for the given test group. |
| email: email address of the owner for the given test group. |
| email_content: email content string (without |appended_text_to_email|) |
| that will be shown on the dashboard. |
| """ |
| # Generate a HTML file that contains all test names for each test group. |
| escaped_tg_name = test_group_name.replace('/', '_') |
| for tg in ['whole', 'skip', 'nonskip']: |
| file_name = os.path.join( |
| os.path.dirname(dashboard_file_location), |
| escaped_tg_name + '_' + tg + '.html') |
| file_object = open(file_name, 'wb') |
| file_object.write('<table border="1">') |
| sorted_testnames = data_map[tg][0].keys() |
| sorted_testnames.sort() |
| for testname in sorted_testnames: |
| file_object.write(( |
| '<tr><td><a href="%s">%s</a></td><td><a href="%s">dashboard</a>' |
| '</td><td>%s</td></tr>') % ( |
| layouttest_root_path + testname, testname, |
| ('http://test-results.appspot.com/dashboards/' |
| 'flakiness_dashboard.html#tests=%s') % testname, |
| data_map[tg][0][testname])) |
| file_object.write('</table>') |
| file_object.close() |
| email_content_with_link = '' |
| if email_content: |
| file_name = os.path.join(os.path.dirname(dashboard_file_location), |
| escaped_tg_name + '_email.html') |
| file_object = open(file_name, 'wb') |
| file_object.write(email_content) |
| file_object.close() |
| email_content_with_link = '<a href="%s_email.html">info</a>' % ( |
| escaped_tg_name) |
| test_group_str = ( |
| '<td><a href="%(test_group_path)s">%(test_group_name)s</a></td>' |
| '<td><a href="%(graph_path)s">graph</a></td>' |
| '<td><a href="%(all_tests_path)s">%(all_tests_count)d</a></td>' |
| '<td><a href="%(skip_tests_path)s">%(skip_tests_count)d</a></td>' |
| '<td><a href="%(nonskip_tests_path)s">%(nonskip_tests_count)d</a></td>' |
| '<td>%(fail_rate)d%%</td>' |
| '<td>%(passing_rate)d%%</td>' |
| '<td><a href="%(rev_url)s">%(rev)s</a></td>' |
| '<td>%(rev_date)s</td>' |
| '<td><a href="mailto:%(email)s">%(email)s</a></td>' |
| '<td>%(email_content)s</td>\n') % { |
| # Dashboard file and graph must be in the same directory |
| # to make the following link work. |
| 'test_group_path': layouttest_root_path + '/' + test_group_name, |
| 'test_group_name': test_group_name, |
| 'graph_path': escaped_tg_name + '.html', |
| 'all_tests_path': escaped_tg_name + '_whole.html', |
| 'all_tests_count': len(data_map['whole'][0]), |
| 'skip_tests_path': escaped_tg_name + '_skip.html', |
| 'skip_tests_count': len(data_map['skip'][0]), |
| 'nonskip_tests_path': escaped_tg_name + '_nonskip.html', |
| 'nonskip_tests_count': len(data_map['nonskip'][0]), |
| 'fail_rate': 100 - float(data_map['passingrate'][0]), |
| 'passing_rate': float(data_map['passingrate'][0]), |
| 'rev_url': DEFAULT_REVISION_VIEW_URL % rev, |
| 'rev': rev, |
| 'rev_date': rev_date, |
| 'email': email, |
| 'email_content': email_content_with_link |
| } |
| layouttest_analyzer_helpers.ReplaceLineInFile( |
| dashboard_file_location, '<td>' + test_group_name + '</td>', |
| test_group_str) |
| |
| |
| def main(): |
| """A main function for the analyzer.""" |
| options = ParseOption() |
| start_time = datetime.now() |
| |
| (prev_time, prev_analyzer_result_map, analyzer_result_map) = ( |
| GetCurrentAndPreviousResults(options.debug, |
| options.test_group_file_location, |
| options.test_group_name, |
| options.result_directory_location)) |
| (result_change, diff_map, simple_rev_str, rev, rev_date, email_content) = ( |
| SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, |
| DEFAULT_EMAIL_APPEND_TEXT, |
| options.email_only_change_mode, options.debug, |
| options.receiver_email_address, options.test_group_name, |
| options.issue_detail_mode)) |
| |
| # Create CSV texts and save them for bug spreadsheet. |
| (stats, issues_txt) = analyzer_result_map.ConvertToCSVText( |
| start_time.strftime('%Y-%m-%d-%H')) |
| file_object = open(os.path.join(options.result_directory_location, |
| DEFAULT_STATS_CSV_FILENAME), 'wb') |
| file_object.write(stats) |
| file_object.close() |
| file_object = open(os.path.join(options.result_directory_location, |
| DEFAULT_ISSUES_CSV_FILENAME), 'wb') |
| file_object.write(issues_txt) |
| file_object.close() |
| |
| if not options.debug and (result_change or not prev_analyzer_result_map): |
| # Save the current result when result is changed or the script is |
| # executed for the first time. |
| date = start_time.strftime('%Y-%m-%d-%H') |
| file_path = os.path.join(options.result_directory_location, date) |
| analyzer_result_map.Save(file_path) |
| if result_change or not prev_analyzer_result_map: |
| data_map = UpdateTrendGraph(start_time, analyzer_result_map, diff_map, |
| simple_rev_str, options.trend_graph_location) |
| # Report the result to dashboard. |
| if options.dashboard_file_location: |
| UpdateDashboard(options.dashboard_file_location, options.test_group_name, |
| data_map, layouttests.DEFAULT_LAYOUTTEST_LOCATION, rev, |
| rev_date, options.receiver_email_address, |
| email_content) |
| |
| |
| if '__main__' == __name__: |
| main() |
