| # Copyright (c) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """Starts a local HTTP server which displays layout test failures (given a test |
| results directory), provides comparisons of expected and actual results (both |
| images and text) and allows one-click rebaselining of tests.""" |
| from __future__ import with_statement |
| |
| import codecs |
| import datetime |
| import fnmatch |
| import mimetypes |
| import os |
| import os.path |
| import shutil |
| import threading |
| import time |
| import urlparse |
| import BaseHTTPServer |
| |
| from optparse import make_option |
| from wsgiref.handlers import format_date_time |
| |
| from webkitpy.common import system |
| from webkitpy.layout_tests.layout_package import json_results_generator |
| from webkitpy.layout_tests.port import factory |
| from webkitpy.layout_tests.port.webkit import WebKitPort |
| from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand |
| from webkitpy.thirdparty import simplejson |
| |
| STATE_NEEDS_REBASELINE = 'needs_rebaseline' |
| STATE_REBASELINE_FAILED = 'rebaseline_failed' |
| STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded' |
| |
| class RebaselineHTTPServer(BaseHTTPServer.HTTPServer): |
| def __init__(self, httpd_port, test_config, results_json, platforms_json): |
| BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler) |
| self.test_config = test_config |
| self.results_json = results_json |
| self.platforms_json = platforms_json |
| |
| |
| class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): |
| STATIC_FILE_NAMES = frozenset([ |
| "index.html", |
| "loupe.js", |
| "main.js", |
| "main.css", |
| "queue.js", |
| "util.js", |
| ]) |
| |
| STATIC_FILE_DIRECTORY = os.path.join( |
| os.path.dirname(__file__), "data", "rebaselineserver") |
| |
| def do_GET(self): |
| self._handle_request() |
| |
| def do_POST(self): |
| self._handle_request() |
| |
| def _handle_request(self): |
| # Parse input. |
| if "?" in self.path: |
| path, query_string = self.path.split("?", 1) |
| self.query = urlparse.parse_qs(query_string) |
| else: |
| path = self.path |
| self.query = {} |
| function_or_file_name = path[1:] or "index.html" |
| |
| # See if a static file matches. |
| if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES: |
| self._serve_static_file(function_or_file_name) |
| return |
| |
| # See if a class method matches. |
| function_name = function_or_file_name.replace(".", "_") |
| if not hasattr(self, function_name): |
| self.send_error(404, "Unknown function %s" % function_name) |
| return |
| if function_name[0] == "_": |
| self.send_error( |
| 401, "Not allowed to invoke private or protected methods") |
| return |
| function = getattr(self, function_name) |
| function() |
| |
| def _serve_static_file(self, static_path): |
| self._serve_file(os.path.join( |
| RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path)) |
| |
| def rebaseline(self): |
| test = self.query['test'][0] |
| baseline_target = self.query['baseline-target'][0] |
| baseline_move_to = self.query['baseline-move-to'][0] |
| test_json = self.server.results_json['tests'][test] |
| |
| if test_json['state'] != STATE_NEEDS_REBASELINE: |
| self.send_error(400, "Test %s is in unexpected state: %s" % |
| (test, test_json["state"])) |
| return |
| |
| log = [] |
| success = _rebaseline_test( |
| test, |
| baseline_target, |
| baseline_move_to, |
| self.server.test_config, |
| log=lambda l: log.append(l)) |
| |
| if success: |
| test_json['state'] = STATE_REBASELINE_SUCCEEDED |
| self.send_response(200) |
| else: |
| test_json['state'] = STATE_REBASELINE_FAILED |
| self.send_response(500) |
| |
| self.send_header('Content-type', 'text/plain') |
| self.end_headers() |
| self.wfile.write('\n'.join(log)) |
| |
| def quitquitquit(self): |
| self.send_response(200) |
| self.send_header("Content-type", "text/plain") |
| self.end_headers() |
| self.wfile.write("Quit.\n") |
| |
| # Shutdown has to happen on another thread from the server's thread, |
| # otherwise there's a deadlock |
| threading.Thread(target=lambda: self.server.shutdown()).start() |
| |
| def test_result(self): |
| test_name, _ = os.path.splitext(self.query['test'][0]) |
| mode = self.query['mode'][0] |
| if mode == 'expected-image': |
| file_name = test_name + '-expected.png' |
| elif mode == 'actual-image': |
| file_name = test_name + '-actual.png' |
| if mode == 'expected-checksum': |
| file_name = test_name + '-expected.checksum' |
| elif mode == 'actual-checksum': |
| file_name = test_name + '-actual.checksum' |
| elif mode == 'diff-image': |
| file_name = test_name + '-diff.png' |
| if mode == 'expected-text': |
| file_name = test_name + '-expected.txt' |
| elif mode == 'actual-text': |
| file_name = test_name + '-actual.txt' |
| elif mode == 'diff-text': |
| file_name = test_name + '-diff.txt' |
| elif mode == 'diff-text-pretty': |
| file_name = test_name + '-pretty-diff.html' |
| |
| file_path = os.path.join(self.server.test_config.results_directory, file_name) |
| |
| # Let results be cached for 60 seconds, so that they can be pre-fetched |
| # by the UI |
| self._serve_file(file_path, cacheable_seconds=60) |
| |
| def results_json(self): |
| self._serve_json(self.server.results_json) |
| |
| def platforms_json(self): |
| self._serve_json(self.server.platforms_json) |
| |
| def _serve_json(self, json): |
| self.send_response(200) |
| self.send_header('Content-type', 'application/json') |
| self.end_headers() |
| simplejson.dump(json, self.wfile) |
| |
| def _serve_file(self, file_path, cacheable_seconds=0): |
| if not os.path.exists(file_path): |
| self.send_error(404, "File not found") |
| return |
| with codecs.open(file_path, "rb") as static_file: |
| self.send_response(200) |
| self.send_header("Content-Length", os.path.getsize(file_path)) |
| mime_type, encoding = mimetypes.guess_type(file_path) |
| if mime_type: |
| self.send_header("Content-type", mime_type) |
| |
| if cacheable_seconds: |
| expires_time = (datetime.datetime.now() + |
| datetime.timedelta(0, cacheable_seconds)) |
| expires_formatted = format_date_time( |
| time.mktime(expires_time.timetuple())) |
| self.send_header("Expires", expires_formatted) |
| self.end_headers() |
| |
| shutil.copyfileobj(static_file, self.wfile) |
| |
| |
| class TestConfig(object): |
| def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm): |
| self.test_port = test_port |
| self.layout_tests_directory = layout_tests_directory |
| self.results_directory = results_directory |
| self.platforms = platforms |
| self.filesystem = filesystem |
| self.scm = scm |
| |
| |
| def _get_actual_result_files(test_file, test_config): |
| test_name, _ = os.path.splitext(test_file) |
| test_directory = os.path.dirname(test_file) |
| |
| test_results_directory = test_config.filesystem.join( |
| test_config.results_directory, test_directory) |
| actual_pattern = os.path.basename(test_name) + '-actual.*' |
| actual_files = [] |
| for filename in test_config.filesystem.listdir(test_results_directory): |
| if fnmatch.fnmatch(filename, actual_pattern): |
| actual_files.append(filename) |
| actual_files.sort() |
| return tuple(actual_files) |
| |
| |
| def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log): |
| test_name, _ = os.path.splitext(test_file) |
| test_directory = os.path.dirname(test_name) |
| |
| log('Rebaselining %s...' % test_name) |
| |
| actual_result_files = _get_actual_result_files(test_file, test_config) |
| filesystem = test_config.filesystem |
| scm = test_config.scm |
| layout_tests_directory = test_config.layout_tests_directory |
| results_directory = test_config.results_directory |
| target_expectations_directory = filesystem.join( |
| layout_tests_directory, 'platform', baseline_target, test_directory) |
| test_results_directory = test_config.filesystem.join( |
| test_config.results_directory, test_directory) |
| |
| # If requested, move current baselines out |
| current_baselines = _get_test_baselines(test_file, test_config) |
| if baseline_target in current_baselines and baseline_move_to != 'none': |
| log(' Moving current %s baselines to %s' % |
| (baseline_target, baseline_move_to)) |
| |
| # See which ones we need to move (only those that are about to be |
| # updated), and make sure we're not clobbering any files in the |
| # destination. |
| current_extensions = set(current_baselines[baseline_target].keys()) |
| actual_result_extensions = [ |
| os.path.splitext(f)[1] for f in actual_result_files] |
| extensions_to_move = current_extensions.intersection( |
| actual_result_extensions) |
| |
| if extensions_to_move.intersection( |
| current_baselines.get(baseline_move_to, {}).keys()): |
| log(' Already had baselines in %s, could not move existing ' |
| '%s ones' % (baseline_move_to, baseline_target)) |
| return False |
| |
| # Do the actual move. |
| if extensions_to_move: |
| if not _move_test_baselines( |
| test_file, |
| list(extensions_to_move), |
| baseline_target, |
| baseline_move_to, |
| test_config, |
| log): |
| return False |
| else: |
| log(' No current baselines to move') |
| |
| log(' Updating baselines for %s' % baseline_target) |
| filesystem.maybe_make_directory(target_expectations_directory) |
| for source_file in actual_result_files: |
| source_path = filesystem.join(test_results_directory, source_file) |
| destination_file = source_file.replace('-actual', '-expected') |
| destination_path = filesystem.join( |
| target_expectations_directory, destination_file) |
| filesystem.copyfile(source_path, destination_path) |
| exit_code = scm.add(destination_path, return_exit_code=True) |
| if exit_code: |
| log(' Could not update %s in SCM, exit code %d' % |
| (destination_file, exit_code)) |
| return False |
| else: |
| log(' Updated %s' % destination_file) |
| |
| return True |
| |
| |
| def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log): |
| test_file_name = os.path.splitext(os.path.basename(test_file))[0] |
| test_directory = os.path.dirname(test_file) |
| filesystem = test_config.filesystem |
| |
| # Want predictable output order for unit tests. |
| extensions_to_move.sort() |
| |
| source_directory = os.path.join( |
| test_config.layout_tests_directory, |
| 'platform', |
| source_platform, |
| test_directory) |
| destination_directory = os.path.join( |
| test_config.layout_tests_directory, |
| 'platform', |
| destination_platform, |
| test_directory) |
| filesystem.maybe_make_directory(destination_directory) |
| |
| for extension in extensions_to_move: |
| file_name = test_file_name + '-expected' + extension |
| source_path = filesystem.join(source_directory, file_name) |
| destination_path = filesystem.join(destination_directory, file_name) |
| filesystem.copyfile(source_path, destination_path) |
| exit_code = test_config.scm.add(destination_path, return_exit_code=True) |
| if exit_code: |
| log(' Could not update %s in SCM, exit code %d' % |
| (file_name, exit_code)) |
| return False |
| else: |
| log(' Moved %s' % file_name) |
| |
| return True |
| |
| def _get_test_baselines(test_file, test_config): |
| class AllPlatformsPort(WebKitPort): |
| def __init__(self): |
| WebKitPort.__init__(self, filesystem=test_config.filesystem) |
| self._platforms_by_directory = dict( |
| [(self._webkit_baseline_path(p), p) for p in test_config.platforms]) |
| |
| def baseline_search_path(self): |
| return self._platforms_by_directory.keys() |
| |
| def platform_from_directory(self, directory): |
| return self._platforms_by_directory[directory] |
| |
| test_path = test_config.filesystem.join( |
| test_config.layout_tests_directory, test_file) |
| |
| all_platforms_port = AllPlatformsPort() |
| |
| all_test_baselines = {} |
| for baseline_extension in ('.txt', '.checksum', '.png'): |
| test_baselines = test_config.test_port.expected_baselines( |
| test_path, baseline_extension) |
| baselines = all_platforms_port.expected_baselines( |
| test_path, baseline_extension, all_baselines=True) |
| for platform_directory, expected_filename in baselines: |
| if not platform_directory: |
| continue |
| if platform_directory == test_config.layout_tests_directory: |
| platform = 'base' |
| else: |
| platform = all_platforms_port.platform_from_directory( |
| platform_directory) |
| platform_baselines = all_test_baselines.setdefault(platform, {}) |
| was_used_for_test = ( |
| platform_directory, expected_filename) in test_baselines |
| platform_baselines[baseline_extension] = was_used_for_test |
| |
| return all_test_baselines |
| |
| |
| class RebaselineServer(AbstractDeclarativeCommand): |
| name = "rebaseline-server" |
| help_text = __doc__ |
| argument_names = "/path/to/results/directory" |
| |
| def __init__(self): |
| options = [ |
| make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"), |
| ] |
| AbstractDeclarativeCommand.__init__(self, options=options) |
| |
| def execute(self, options, args, tool): |
| results_directory = args[0] |
| filesystem = system.filesystem.FileSystem() |
| scm = self._tool.scm() |
| |
| if options.dry_run: |
| |
| def no_op_copyfile(src, dest): |
| pass |
| |
| def no_op_add(path, return_exit_code=False): |
| if return_exit_code: |
| return 0 |
| |
| filesystem.copyfile = no_op_copyfile |
| scm.add = no_op_add |
| |
| print 'Parsing unexpected_results.json...' |
| results_json_path = filesystem.join(results_directory, 'unexpected_results.json') |
| results_json = json_results_generator.load_json(filesystem, results_json_path) |
| |
| port = factory.get() |
| layout_tests_directory = port.layout_tests_dir() |
| platforms = filesystem.listdir( |
| filesystem.join(layout_tests_directory, 'platform')) |
| test_config = TestConfig( |
| port, |
| layout_tests_directory, |
| results_directory, |
| platforms, |
| filesystem, |
| scm) |
| |
| print 'Gathering current baselines...' |
| for test_file, test_json in results_json['tests'].items(): |
| test_json['state'] = STATE_NEEDS_REBASELINE |
| test_path = filesystem.join(layout_tests_directory, test_file) |
| test_json['baselines'] = _get_test_baselines(test_file, test_config) |
| |
| server_url = "http://localhost:%d/" % options.httpd_port |
| print "Starting server at %s" % server_url |
| print ("Use the 'Exit' link in the UI, %squitquitquit " |
| "or Ctrl-C to stop") % server_url |
| |
| threading.Timer( |
| .1, lambda: self._tool.user.open_url(server_url)).start() |
| |
| httpd = RebaselineHTTPServer( |
| httpd_port=options.httpd_port, |
| test_config=test_config, |
| results_json=results_json, |
| platforms_json={ |
| 'platforms': platforms, |
| 'defaultPlatform': port.name(), |
| }) |
| httpd.serve_forever() |
