| #!/usr/bin/python |
| # -*- coding: utf-8 -*- |
| |
| from __future__ import print_function |
| import argparse |
| import BaseHTTPServer |
| import json |
| import os |
| import os.path |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import urllib2 |
| |
| # Grab the script path because that is where all the static assets are |
| SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| |
| # Find the tools directory for python imports |
| TOOLS_DIR = os.path.dirname(SCRIPT_DIR) |
| |
| # Find the root of the skia trunk for finding skpdiff binary |
| SKIA_ROOT_DIR = os.path.dirname(TOOLS_DIR) |
| |
| # Find the default location of gm expectations |
| DEFAULT_GM_EXPECTATIONS_DIR = os.path.join(SKIA_ROOT_DIR, 'expectations', 'gm') |
| |
| # Imports from within Skia |
| if TOOLS_DIR not in sys.path: |
| sys.path.append(TOOLS_DIR) |
| GM_DIR = os.path.join(SKIA_ROOT_DIR, 'gm') |
| if GM_DIR not in sys.path: |
| sys.path.append(GM_DIR) |
| import gm_json |
| import jsondiff |
| |
| # A simple dictionary of file name extensions to MIME types. The empty string |
| # entry is used as the default when no extension was given or if the extension |
| # has no entry in this dictionary. |
| MIME_TYPE_MAP = {'': 'application/octet-stream', |
| 'html': 'text/html', |
| 'css': 'text/css', |
| 'png': 'image/png', |
| 'js': 'application/javascript', |
| 'json': 'application/json' |
| } |
| |
| |
| IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
| |
| SKPDIFF_INVOKE_FORMAT = '{} --jsonp=false -o {} -f {} {}' |
| |
| |
| def get_skpdiff_path(user_path=None): |
| """Find the skpdiff binary. |
| |
| @param user_path If none, searches in Release and Debug out directories of |
| the skia root. If set, checks that the path is a real file and |
| returns it. |
| """ |
| skpdiff_path = None |
| possible_paths = [] |
| |
| # Use the user given path, or try out some good default paths. |
| if user_path: |
| possible_paths.append(user_path) |
| else: |
| possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', |
| 'Release', 'skpdiff')) |
| possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', |
| 'Release', 'skpdiff.exe')) |
| possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', |
| 'Debug', 'skpdiff')) |
| possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', |
| 'Debug', 'skpdiff.exe')) |
| # Use the first path that actually points to the binary |
| for possible_path in possible_paths: |
| if os.path.isfile(possible_path): |
| skpdiff_path = possible_path |
| break |
| |
| # If skpdiff was not found, print out diagnostic info for the user. |
| if skpdiff_path is None: |
| print('Could not find skpdiff binary. Either build it into the ' + |
| 'default directory, or specify the path on the command line.') |
| print('skpdiff paths tried:') |
| for possible_path in possible_paths: |
| print(' ', possible_path) |
| return skpdiff_path |
| |
| |
| def download_file(url, output_path): |
| """Download the file at url and place it in output_path""" |
| reader = urllib2.urlopen(url) |
| with open(output_path, 'wb') as writer: |
| writer.write(reader.read()) |
| |
| |
| def download_gm_image(image_name, image_path, hash_val): |
| """Download the gm result into the given path. |
| |
| @param image_name The GM file name, for example imageblur_gpu.png. |
| @param image_path Path to place the image. |
| @param hash_val The hash value of the image. |
| """ |
| if hash_val is None: |
| return |
| |
| # Separate the test name from a image name |
| image_match = IMAGE_FILENAME_RE.match(image_name) |
| test_name = image_match.group(1) |
| |
| # Calculate the URL of the requested image |
| image_url = gm_json.CreateGmActualUrl( |
| test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val) |
| |
| # Download the image as requested |
| download_file(image_url, image_path) |
| |
| |
| def get_image_set_from_skpdiff(skpdiff_records): |
| """Get the set of all images references in the given records. |
| |
| @param skpdiff_records An array of records, which are dictionary objects. |
| """ |
| expected_set = frozenset([r['baselinePath'] for r in skpdiff_records]) |
| actual_set = frozenset([r['testPath'] for r in skpdiff_records]) |
| return expected_set | actual_set |
| |
| |
| def set_expected_hash_in_json(expected_results_json, image_name, hash_value): |
| """Set the expected hash for the object extracted from |
| expected-results.json. Note that this only work with bitmap-64bitMD5 hash |
| types. |
| |
| @param expected_results_json The Python dictionary with the results to |
| modify. |
| @param image_name The name of the image to set the hash of. |
| @param hash_value The hash to set for the image. |
| """ |
| expected_results = expected_results_json[gm_json.JSONKEY_EXPECTEDRESULTS] |
| |
| if image_name in expected_results: |
| expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0][1] = hash_value |
| else: |
| expected_results[image_name] = { |
| gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: |
| [ |
| [ |
| gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, |
| hash_value |
| ] |
| ] |
| } |
| |
| |
| def get_head_version(path): |
| """Get the version of the file at the given path stored inside the HEAD of |
| the git repository. It is returned as a string. |
| |
| @param path The path of the file whose HEAD is returned. It is assumed the |
| path is inside a git repo rooted at SKIA_ROOT_DIR. |
| """ |
| |
| # git-show will not work with absolute paths. This ensures we give it a path |
| # relative to the skia root. This path also has to use forward slashes, even |
| # on windows. |
| git_path = os.path.relpath(path, SKIA_ROOT_DIR).replace('\\', '/') |
| git_show_proc = subprocess.Popen(['git', 'show', 'HEAD:' + git_path], |
| stdout=subprocess.PIPE) |
| |
| # When invoked outside a shell, git will output the last committed version |
| # of the file directly to stdout. |
| git_version_content, _ = git_show_proc.communicate() |
| return git_version_content |
| |
| |
| class GMInstance: |
| """Information about a GM test result on a specific device: |
| - device_name = the name of the device that rendered it |
| - image_name = the GM test name and config |
| - expected_hash = the current expected hash value |
| - actual_hash = the actual hash value |
| - is_rebaselined = True if actual_hash is what is currently in the expected |
| results file, False otherwise. |
| """ |
| def __init__(self, |
| device_name, image_name, |
| expected_hash, actual_hash, |
| is_rebaselined): |
| self.device_name = device_name |
| self.image_name = image_name |
| self.expected_hash = expected_hash |
| self.actual_hash = actual_hash |
| self.is_rebaselined = is_rebaselined |
| |
| |
| class ExpectationsManager: |
| def __init__(self, expectations_dir, expected_name, updated_name, |
| skpdiff_path): |
| """ |
| @param expectations_dir The directory to traverse for results files. |
| This should resemble expectations/gm in the Skia trunk. |
| @param expected_name The name of the expected result files. These |
| are in the format of expected-results.json. |
| @param updated_name The name of the updated expected result files. |
| Normally this matches --expectations-filename-output for the |
| rebaseline.py tool. |
| @param skpdiff_path The path used to execute the skpdiff command. |
| """ |
| self._expectations_dir = expectations_dir |
| self._expected_name = expected_name |
| self._updated_name = updated_name |
| self._skpdiff_path = skpdiff_path |
| self._generate_gm_comparison() |
| |
| def _generate_gm_comparison(self): |
| """Generate all the data needed to compare GMs: |
| - determine which GMs changed |
| - download the changed images |
| - compare them with skpdiff |
| """ |
| |
| # Get the expectations and compare them with actual hashes |
| self._get_expectations() |
| |
| |
| # Create a temporary file tree that makes sense for skpdiff to operate |
| # on. We take the realpath of the new temp directory because some OSs |
| # (*cough* osx) put the temp directory behind a symlink that gets |
| # resolved later down the pipeline and breaks the image map. |
| image_output_dir = os.path.realpath(tempfile.mkdtemp('skpdiff')) |
| expected_image_dir = os.path.join(image_output_dir, 'expected') |
| actual_image_dir = os.path.join(image_output_dir, 'actual') |
| os.mkdir(expected_image_dir) |
| os.mkdir(actual_image_dir) |
| |
| # Download expected and actual images that differed into the temporary |
| # file tree. |
| self._download_expectation_images(expected_image_dir, actual_image_dir) |
| |
| # Invoke skpdiff with our downloaded images and place its results in the |
| # temporary directory. |
| self._skpdiff_output_path = os.path.join(image_output_dir, |
| 'skpdiff_output.json') |
| skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path, |
| self._skpdiff_output_path, |
| expected_image_dir, |
| actual_image_dir) |
| os.system(skpdiff_cmd) |
| self._load_skpdiff_output() |
| |
| |
| def _get_expectations(self): |
| """Fills self._expectations with GMInstance objects for each test whose |
| expectation is different between the following two files: |
| - the local filesystem's updated results file |
| - git's head version of the expected results file |
| """ |
| differ = jsondiff.GMDiffer() |
| self._expectations = [] |
| for root, dirs, files in os.walk(self._expectations_dir): |
| for expectation_file in files: |
| # There are many files in the expectations directory. We only |
| # care about expected results. |
| if expectation_file != self._expected_name: |
| continue |
| |
| # Get the name of the results file, and be sure there is an |
| # updated result to compare against. If there is not, there is |
| # no point in diffing this device. |
| expected_file_path = os.path.join(root, self._expected_name) |
| updated_file_path = os.path.join(root, self._updated_name) |
| if not os.path.isfile(updated_file_path): |
| continue |
| |
| # Always get the expected results from git because we may have |
| # changed them in a previous instance of the server. |
| expected_contents = get_head_version(expected_file_path) |
| updated_contents = None |
| with open(updated_file_path, 'rb') as updated_file: |
| updated_contents = updated_file.read() |
| |
| # Read the expected results on disk to determine what we've |
| # already rebaselined. |
| commited_contents = None |
| with open(expected_file_path, 'rb') as expected_file: |
| commited_contents = expected_file.read() |
| |
| # Find all expectations that did not match. |
| expected_diff = differ.GenerateDiffDictFromStrings( |
| expected_contents, |
| updated_contents) |
| |
| # Generate a set of images that have already been rebaselined |
| # onto disk. |
| rebaselined_diff = differ.GenerateDiffDictFromStrings( |
| expected_contents, |
| commited_contents) |
| |
| rebaselined_set = set(rebaselined_diff.keys()) |
| |
| # The name of the device corresponds to the name of the folder |
| # we are in. |
| device_name = os.path.basename(root) |
| |
| # Store old and new versions of the expectation for each GM |
| for image_name, hashes in expected_diff.iteritems(): |
| self._expectations.append( |
| GMInstance(device_name, image_name, |
| hashes['old'], hashes['new'], |
| image_name in rebaselined_set)) |
| |
| def _load_skpdiff_output(self): |
| """Loads the results of skpdiff and annotates them with whether they |
| have already been rebaselined or not. The resulting data is store in |
| self.skpdiff_records.""" |
| self.skpdiff_records = None |
| with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file: |
| self.skpdiff_records = json.load(skpdiff_output_file)['records'] |
| for record in self.skpdiff_records: |
| record['isRebaselined'] = self.image_map[record['baselinePath']][1].is_rebaselined |
| |
| |
| def _download_expectation_images(self, expected_image_dir, actual_image_dir): |
| """Download the expected and actual images for the _expectations array. |
| |
| @param expected_image_dir The directory to download expected images |
| into. |
| @param actual_image_dir The directory to download actual images into. |
| """ |
| image_map = {} |
| |
| # Look through expectations and download their images. |
| for expectation in self._expectations: |
| # Build appropriate paths to download the images into. |
| expected_image_path = os.path.join(expected_image_dir, |
| expectation.device_name + '-' + |
| expectation.image_name) |
| |
| actual_image_path = os.path.join(actual_image_dir, |
| expectation.device_name + '-' + |
| expectation.image_name) |
| |
| print('Downloading %s for device %s' % ( |
| expectation.image_name, expectation.device_name)) |
| |
| # Download images |
| download_gm_image(expectation.image_name, |
| expected_image_path, |
| expectation.expected_hash) |
| |
| download_gm_image(expectation.image_name, |
| actual_image_path, |
| expectation.actual_hash) |
| |
| # Annotate the expectations with where the images were downloaded |
| # to. |
| expectation.expected_image_path = expected_image_path |
| expectation.actual_image_path = actual_image_path |
| |
| # Map the image paths back to the expectations. |
| image_map[expected_image_path] = (False, expectation) |
| image_map[actual_image_path] = (True, expectation) |
| |
| self.image_map = image_map |
| |
| def _set_expected_hash(self, device_name, image_name, hash_value): |
| """Set the expected hash for the image of the given device. This always |
| writes directly to the expected results file of the given device |
| |
| @param device_name The name of the device to write the hash to. |
| @param image_name The name of the image whose hash to set. |
| @param hash_value The value of the hash to set. |
| """ |
| |
| # Retrieve the expected results file as it is in the working tree |
| json_path = os.path.join(self._expectations_dir, device_name, |
| self._expected_name) |
| expectations = gm_json.LoadFromFile(json_path) |
| |
| # Set the specified hash. |
| set_expected_hash_in_json(expectations, image_name, hash_value) |
| |
| # Write it out to disk using gm_json to keep the formatting consistent. |
| gm_json.WriteToFile(expectations, json_path) |
| |
| def commit_rebaselines(self, rebaselines): |
| """Sets the expected results file to use the hashes of the images in |
| the rebaselines list. If a expected result image is not in rebaselines |
| at all, the old hash will be used. |
| |
| @param rebaselines A list of image paths to use the hash of. |
| """ |
| # Reset all expectations to their old hashes because some of them may |
| # have been set to the new hash by a previous call to this function. |
| for expectation in self._expectations: |
| expectation.is_rebaselined = False |
| self._set_expected_hash(expectation.device_name, |
| expectation.image_name, |
| expectation.expected_hash) |
| |
| # Take all the images to rebaseline |
| for image_path in rebaselines: |
| # Get the metadata about the image at the path. |
| is_actual, expectation = self.image_map[image_path] |
| |
| expectation.is_rebaselined = is_actual |
| expectation_hash = expectation.actual_hash if is_actual else\ |
| expectation.expected_hash |
| |
| # Write out that image's hash directly to the expected results file. |
| self._set_expected_hash(expectation.device_name, |
| expectation.image_name, |
| expectation_hash) |
| |
| self._load_skpdiff_output() |
| |
| |
| class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler): |
| def send_file(self, file_path): |
| # Grab the extension if there is one |
| extension = os.path.splitext(file_path)[1] |
| if len(extension) >= 1: |
| extension = extension[1:] |
| |
| # Determine the MIME type of the file from its extension |
| mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP['']) |
| |
| # Open the file and send it over HTTP |
| if os.path.isfile(file_path): |
| with open(file_path, 'rb') as sending_file: |
| self.send_response(200) |
| self.send_header('Content-type', mime_type) |
| self.end_headers() |
| self.wfile.write(sending_file.read()) |
| else: |
| self.send_error(404) |
| |
| def serve_if_in_dir(self, dir_path, file_path): |
| # Determine if the file exists relative to the given dir_path AND exists |
| # under the dir_path. This is to prevent accidentally serving files |
| # outside the directory intended using symlinks, or '../'. |
| real_path = os.path.normpath(os.path.join(dir_path, file_path)) |
| if os.path.commonprefix([real_path, dir_path]) == dir_path: |
| if os.path.isfile(real_path): |
| self.send_file(real_path) |
| return True |
| return False |
| |
| def do_GET(self): |
| # Simple rewrite rule of the root path to 'viewer.html' |
| if self.path == '' or self.path == '/': |
| self.path = '/viewer.html' |
| |
| # The [1:] chops off the leading '/' |
| file_path = self.path[1:] |
| |
| # Handle skpdiff_output.json manually because it is was processed by the |
| # server when it was started and does not exist as a file. |
| if file_path == 'skpdiff_output.json': |
| self.send_response(200) |
| self.send_header('Content-type', MIME_TYPE_MAP['json']) |
| self.end_headers() |
| |
| # Add JSONP padding to the JSON because the web page expects it. It |
| # expects it because it was designed to run with or without a web |
| # server. Without a web server, the only way to load JSON is with |
| # JSONP. |
| skpdiff_records = self.server.expectations_manager.skpdiff_records |
| self.wfile.write('var SkPDiffRecords = ') |
| json.dump({'records': skpdiff_records}, self.wfile) |
| self.wfile.write(';') |
| return |
| |
| # Attempt to send static asset files first. |
| if self.serve_if_in_dir(SCRIPT_DIR, file_path): |
| return |
| |
| # WARNING: Serving any file the user wants is incredibly insecure. Its |
| # redeeming quality is that we only serve gm files on a white list. |
| if self.path in self.server.image_set: |
| self.send_file(self.path) |
| return |
| |
| # If no file to send was found, just give the standard 404 |
| self.send_error(404) |
| |
| def do_POST(self): |
| if self.path == '/commit_rebaselines': |
| content_length = int(self.headers['Content-length']) |
| request_data = json.loads(self.rfile.read(content_length)) |
| rebaselines = request_data['rebaselines'] |
| self.server.expectations_manager.commit_rebaselines(rebaselines) |
| self.send_response(200) |
| self.send_header('Content-type', 'application/json') |
| self.end_headers() |
| self.wfile.write('{"success":true}') |
| return |
| |
| # If the we have no handler for this path, give em' the 404 |
| self.send_error(404) |
| |
| |
| def run_server(expectations_manager, port=8080): |
| # It's important to parse the results file so that we can make a set of |
| # images that the web page might request. |
| skpdiff_records = expectations_manager.skpdiff_records |
| image_set = get_image_set_from_skpdiff(skpdiff_records) |
| |
| # Do not bind to interfaces other than localhost because the server will |
| # attempt to serve files relative to the root directory as a last resort |
| # before 404ing. This means all of your files can be accessed from this |
| # server, so DO NOT let this server listen to anything but localhost. |
| server_address = ('127.0.0.1', port) |
| http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler) |
| http_server.image_set = image_set |
| http_server.expectations_manager = expectations_manager |
| print('Navigate thine browser to: http://{}:{}/'.format(*server_address)) |
| http_server.serve_forever() |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--port', '-p', metavar='PORT', |
| type=int, |
| default=8080, |
| help='port to bind the server to; ' + |
| 'defaults to %(default)s', |
| ) |
| |
| parser.add_argument('--expectations-dir', metavar='EXPECTATIONS_DIR', |
| default=DEFAULT_GM_EXPECTATIONS_DIR, |
| help='path to the gm expectations; ' + |
| 'defaults to %(default)s' |
| ) |
| |
| parser.add_argument('--expected', |
| metavar='EXPECTATIONS_FILE_NAME', |
| default='expected-results.json', |
| help='the file name of the expectations JSON; ' + |
| 'defaults to %(default)s' |
| ) |
| |
| parser.add_argument('--updated', |
| metavar='UPDATED_FILE_NAME', |
| default='updated-results.json', |
| help='the file name of the updated expectations JSON;' + |
| ' defaults to %(default)s' |
| ) |
| |
| parser.add_argument('--skpdiff-path', metavar='SKPDIFF_PATH', |
| default=None, |
| help='the path to the skpdiff binary to use; ' + |
| 'defaults to out/Release/skpdiff or out/Default/skpdiff' |
| ) |
| |
| args = vars(parser.parse_args()) # Convert args into a python dict |
| |
| # Make sure we have access to an skpdiff binary |
| skpdiff_path = get_skpdiff_path(args['skpdiff_path']) |
| if skpdiff_path is None: |
| sys.exit(1) |
| |
| # Print out the paths of things for easier debugging |
| print('script dir :', SCRIPT_DIR) |
| print('tools dir :', TOOLS_DIR) |
| print('root dir :', SKIA_ROOT_DIR) |
| print('expectations dir :', args['expectations_dir']) |
| print('skpdiff path :', skpdiff_path) |
| |
| expectations_manager = ExpectationsManager(args['expectations_dir'], |
| args['expected'], |
| args['updated'], |
| skpdiff_path) |
| |
| run_server(expectations_manager, port=args['port']) |
| |
| if __name__ == '__main__': |
| main() |
