| #!/usr/bin/env python |
| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """Run layout tests.""" |
| |
| import errno |
| import logging |
| import optparse |
| import os |
| import signal |
| import sys |
| |
| from layout_package import json_results_generator |
| from layout_package import printing |
| from layout_package import test_runner |
| from layout_package import test_runner2 |
| |
| from webkitpy.common.system import user |
| from webkitpy.thirdparty import simplejson |
| |
| import port |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| def run(port, options, args, regular_output=sys.stderr, |
| buildbot_output=sys.stdout): |
| """Run the tests. |
| |
| Args: |
| port: Port object for port-specific behavior |
| options: a dictionary of command line options |
| args: a list of sub directories or files to test |
| regular_output: a stream-like object that we can send logging/debug |
| output to |
| buildbot_output: a stream-like object that we can write all output that |
| is intended to be parsed by the buildbot to |
| Returns: |
| the number of unexpected results that occurred, or -1 if there is an |
| error. |
| |
| """ |
| warnings = _set_up_derived_options(port, options) |
| |
| printer = printing.Printer(port, options, regular_output, buildbot_output, |
| int(options.child_processes), options.experimental_fully_parallel) |
| for w in warnings: |
| _log.warning(w) |
| |
| if options.help_printing: |
| printer.help_printing() |
| printer.cleanup() |
| return 0 |
| |
| last_unexpected_results = _gather_unexpected_results(port) |
| if options.print_last_failures: |
| printer.write("\n".join(last_unexpected_results) + "\n") |
| printer.cleanup() |
| return 0 |
| |
| # We wrap any parts of the run that are slow or likely to raise exceptions |
| # in a try/finally to ensure that we clean up the logging configuration. |
| num_unexpected_results = -1 |
| try: |
| runner = test_runner2.TestRunner2(port, options, printer) |
| runner._print_config() |
| |
| printer.print_update("Collecting tests ...") |
| try: |
| runner.collect_tests(args, last_unexpected_results) |
| except IOError, e: |
| if e.errno == errno.ENOENT: |
| return -1 |
| raise |
| |
| if options.lint_test_files: |
| return runner.lint() |
| |
| printer.print_update("Parsing expectations ...") |
| runner.parse_expectations() |
| |
| printer.print_update("Checking build ...") |
| if not port.check_build(runner.needs_http()): |
| _log.error("Build check failed") |
| return -1 |
| |
| result_summary = runner.set_up_run() |
| if result_summary: |
| num_unexpected_results = runner.run(result_summary) |
| runner.clean_up_run() |
| _log.debug("Testing completed, Exit status: %d" % |
| num_unexpected_results) |
| finally: |
| printer.cleanup() |
| |
| return num_unexpected_results |
| |
| |
| def _set_up_derived_options(port_obj, options): |
| """Sets the options values that depend on other options values.""" |
| # We return a list of warnings to print after the printer is initialized. |
| warnings = [] |
| |
| if options.worker_model is None: |
| options.worker_model = port_obj.default_worker_model() |
| |
| if options.worker_model == 'inline': |
| if options.child_processes and int(options.child_processes) > 1: |
| warnings.append("--worker-model=inline overrides --child-processes") |
| options.child_processes = "1" |
| if not options.child_processes: |
| options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", |
| str(port_obj.default_child_processes())) |
| |
| if not options.configuration: |
| options.configuration = port_obj.default_configuration() |
| |
| if options.pixel_tests is None: |
| options.pixel_tests = True |
| |
| if not options.use_apache: |
| options.use_apache = sys.platform in ('darwin', 'linux2') |
| |
| if not options.time_out_ms: |
| if options.configuration == "Debug": |
| options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS) |
| else: |
| options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS) |
| |
| options.slow_time_out_ms = str(5 * int(options.time_out_ms)) |
| |
| if options.additional_platform_directory: |
| normalized_platform_directories = [] |
| for path in options.additional_platform_directory: |
| if not port_obj._filesystem.isabs(path): |
| warnings.append("--additional-platform-directory=%s is ignored since it is not absolute" % path) |
| continue |
| normalized_platform_directories.append(port_obj._filesystem.normpath(path)) |
| options.additional_platform_directory = normalized_platform_directories |
| |
| return warnings |
| |
| |
| def _gather_unexpected_results(port): |
| """Returns the unexpected results from the previous run, if any.""" |
| filesystem = port._filesystem |
| results_directory = port.results_directory() |
| options = port._options |
| last_unexpected_results = [] |
| if options.print_last_failures or options.retest_last_failures: |
| unexpected_results_filename = filesystem.join(results_directory, "unexpected_results.json") |
| if filesystem.exists(unexpected_results_filename): |
| results = json_results_generator.load_json(filesystem, unexpected_results_filename) |
| last_unexpected_results = results['tests'].keys() |
| return last_unexpected_results |
| |
| |
| def _compat_shim_callback(option, opt_str, value, parser): |
| print "Ignoring unsupported option: %s" % opt_str |
| |
| |
| def _compat_shim_option(option_name, **kwargs): |
| return optparse.make_option(option_name, action="callback", |
| callback=_compat_shim_callback, |
| help="Ignored, for old-run-webkit-tests compat only.", **kwargs) |
| |
| |
| def parse_args(args=None): |
| """Provides a default set of command line args. |
| |
| Returns a tuple of options, args from optparse""" |
| |
| # FIXME: All of these options should be stored closer to the code which |
| # FIXME: actually uses them. configuration_options should move |
| # FIXME: to WebKitPort and be shared across all scripts. |
| configuration_options = [ |
| optparse.make_option("-t", "--target", dest="configuration", |
| help="(DEPRECATED)"), |
| # FIXME: --help should display which configuration is default. |
| optparse.make_option('--debug', action='store_const', const='Debug', |
| dest="configuration", |
| help='Set the configuration to Debug'), |
| optparse.make_option('--release', action='store_const', |
| const='Release', dest="configuration", |
| help='Set the configuration to Release'), |
| # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. |
| ] |
| |
| print_options = printing.print_options() |
| |
| # FIXME: These options should move onto the ChromiumPort. |
| chromium_options = [ |
| optparse.make_option("--chromium", action="store_true", default=False, |
| help="use the Chromium port"), |
| optparse.make_option("--startup-dialog", action="store_true", |
| default=False, help="create a dialog on DumpRenderTree startup"), |
| optparse.make_option("--gp-fault-error-box", action="store_true", |
| default=False, help="enable Windows GP fault error box"), |
| optparse.make_option("--js-flags", |
| type="string", help="JavaScript flags to pass to tests"), |
| optparse.make_option("--stress-opt", action="store_true", |
| default=False, |
| help="Enable additional stress test to JavaScript optimization"), |
| optparse.make_option("--stress-deopt", action="store_true", |
| default=False, |
| help="Enable additional stress test to JavaScript optimization"), |
| optparse.make_option("--nocheck-sys-deps", action="store_true", |
| default=False, |
| help="Don't check the system dependencies (themes)"), |
| optparse.make_option("--accelerated-compositing", |
| action="store_true", |
| help="Use hardware-accelerated compositing for rendering"), |
| optparse.make_option("--no-accelerated-compositing", |
| action="store_false", |
| dest="accelerated_compositing", |
| help="Don't use hardware-accelerated compositing for rendering"), |
| optparse.make_option("--accelerated-2d-canvas", |
| action="store_true", |
| help="Use hardware-accelerated 2D Canvas calls"), |
| optparse.make_option("--no-accelerated-2d-canvas", |
| action="store_false", |
| dest="accelerated_2d_canvas", |
| help="Don't use hardware-accelerated 2D Canvas calls"), |
| optparse.make_option("--enable-hardware-gpu", |
| action="store_true", |
| default=False, |
| help="Run graphics tests on real GPU hardware vs software"), |
| ] |
| |
| # Missing Mac-specific old-run-webkit-tests options: |
| # FIXME: Need: -g, --guard for guard malloc support on Mac. |
| # FIXME: Need: -l --leaks Enable leaks checking. |
| # FIXME: Need: --sample-on-timeout Run sample on timeout |
| |
| old_run_webkit_tests_compat = [ |
| # NRWT doesn't generate results by default anyway. |
| _compat_shim_option("--no-new-test-results"), |
| # NRWT doesn't sample on timeout yet anyway. |
| _compat_shim_option("--no-sample-on-timeout"), |
| # FIXME: NRWT needs to support remote links eventually. |
| _compat_shim_option("--use-remote-links-to-tests"), |
| ] |
| |
| results_options = [ |
| # NEED for bots: --use-remote-links-to-tests Link to test files |
| # within the SVN repository in the results. |
| optparse.make_option("-p", "--pixel-tests", action="store_true", |
| dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), |
| optparse.make_option("--no-pixel-tests", action="store_false", |
| dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), |
| optparse.make_option("--tolerance", |
| help="Ignore image differences less than this percentage (some " |
| "ports may ignore this option)", type="float"), |
| optparse.make_option("--results-directory", help="Location of test results"), |
| optparse.make_option("--build-directory", |
| help="Path to the directory under which build files are kept (should not include configuration)"), |
| optparse.make_option("--new-baseline", action="store_true", |
| default=False, help="Save all generated results as new baselines " |
| "into the platform directory, overwriting whatever's " |
| "already there."), |
| optparse.make_option("--reset-results", action="store_true", |
| default=False, help="Reset any existing baselines to the " |
| "generated results"), |
| optparse.make_option("--additional-drt-flag", action="append", |
| default=[], help="Additional command line flag to pass to DumpRenderTree " |
| "Specify multiple times to add multiple flags."), |
| optparse.make_option("--additional-platform-directory", action="append", |
| default=[], help="Additional directory where to look for test " |
| "baselines (will take precendence over platform baselines). " |
| "Specify multiple times to add multiple search path entries."), |
| optparse.make_option("--no-show-results", action="store_false", |
| default=True, dest="show_results", |
| help="Don't launch a browser with results after the tests " |
| "are done"), |
| # FIXME: We should have a helper function to do this sort of |
| # deprectated mapping and automatically log, etc. |
| optparse.make_option("--noshow-results", action="store_false", |
| dest="show_results", |
| help="Deprecated, same as --no-show-results."), |
| optparse.make_option("--no-launch-safari", action="store_false", |
| dest="show_results", |
| help="old-run-webkit-tests compat, same as --noshow-results."), |
| # old-run-webkit-tests: |
| # --[no-]launch-safari Launch (or do not launch) Safari to display |
| # test results (default: launch) |
| optparse.make_option("--full-results-html", action="store_true", |
| default=False, |
| help="Show all failures in results.html, rather than only " |
| "regressions"), |
| optparse.make_option("--clobber-old-results", action="store_true", |
| default=False, help="Clobbers test results from previous runs."), |
| optparse.make_option("--platform", |
| help="Override the platform for expected results"), |
| optparse.make_option("--no-record-results", action="store_false", |
| default=True, dest="record_results", |
| help="Don't record the results."), |
| # old-run-webkit-tests also has HTTP toggle options: |
| # --[no-]http Run (or do not run) http tests |
| # (default: run) |
| ] |
| |
| test_options = [ |
| optparse.make_option("--build", dest="build", |
| action="store_true", default=True, |
| help="Check to ensure the DumpRenderTree build is up-to-date " |
| "(default)."), |
| optparse.make_option("--no-build", dest="build", |
| action="store_false", help="Don't check to see if the " |
| "DumpRenderTree build is up-to-date."), |
| optparse.make_option("-n", "--dry-run", action="store_true", |
| default=False, |
| help="Do everything but actually run the tests or upload results."), |
| # old-run-webkit-tests has --valgrind instead of wrapper. |
| optparse.make_option("--wrapper", |
| help="wrapper command to insert before invocations of " |
| "DumpRenderTree; option is split on whitespace before " |
| "running. (Example: --wrapper='valgrind --smc-check=all')"), |
| # old-run-webkit-tests: |
| # -i|--ignore-tests Comma-separated list of directories |
| # or tests to ignore |
| optparse.make_option("--test-list", action="append", |
| help="read list of tests to run from file", metavar="FILE"), |
| # old-run-webkit-tests uses --skipped==[default|ignore|only] |
| # instead of --force: |
| optparse.make_option("--force", action="store_true", default=False, |
| help="Run all tests, even those marked SKIP in the test list"), |
| optparse.make_option("--use-apache", action="store_true", |
| default=False, help="Whether to use apache instead of lighttpd."), |
| optparse.make_option("--time-out-ms", |
| help="Set the timeout for each test"), |
| # old-run-webkit-tests calls --randomize-order --random: |
| optparse.make_option("--randomize-order", action="store_true", |
| default=False, help=("Run tests in random order (useful " |
| "for tracking down corruption)")), |
| optparse.make_option("--run-chunk", |
| help=("Run a specified chunk (n:l), the nth of len l, " |
| "of the layout tests")), |
| optparse.make_option("--run-part", help=("Run a specified part (n:m), " |
| "the nth of m parts, of the layout tests")), |
| # old-run-webkit-tests calls --batch-size: --nthly n |
| # Restart DumpRenderTree every n tests (default: 1000) |
| optparse.make_option("--batch-size", |
| help=("Run a the tests in batches (n), after every n tests, " |
| "DumpRenderTree is relaunched."), type="int", default=0), |
| # old-run-webkit-tests calls --run-singly: -1|--singly |
| # Isolate each test case run (implies --nthly 1 --verbose) |
| optparse.make_option("--run-singly", action="store_true", |
| default=False, help="run a separate DumpRenderTree for each test"), |
| optparse.make_option("--child-processes", |
| help="Number of DumpRenderTrees to run in parallel."), |
| # FIXME: Display default number of child processes that will run. |
| optparse.make_option("--worker-model", action="store", |
| default=None, help=("controls worker model. Valid values are " |
| "'inline', 'threads', and 'processes'.")), |
| optparse.make_option("--experimental-fully-parallel", |
| action="store_true", default=False, |
| help="run all tests in parallel"), |
| optparse.make_option("--exit-after-n-failures", type="int", default=500, |
| help="Exit after the first N failures instead of running all " |
| "tests"), |
| optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", |
| default=20, help="Exit after the first N crashes instead of " |
| "running all tests"), |
| # FIXME: consider: --iterations n |
| # Number of times to run the set of tests (e.g. ABCABCABC) |
| optparse.make_option("--print-last-failures", action="store_true", |
| default=False, help="Print the tests in the last run that " |
| "had unexpected failures (or passes) and then exit."), |
| optparse.make_option("--retest-last-failures", action="store_true", |
| default=False, help="re-test the tests in the last run that " |
| "had unexpected failures (or passes)."), |
| optparse.make_option("--retry-failures", action="store_true", |
| default=True, |
| help="Re-try any tests that produce unexpected results (default)"), |
| optparse.make_option("--no-retry-failures", action="store_false", |
| dest="retry_failures", |
| help="Don't re-try any tests that produce unexpected results."), |
| ] |
| |
| misc_options = [ |
| optparse.make_option("--lint-test-files", action="store_true", |
| default=False, help=("Makes sure the test files parse for all " |
| "configurations. Does not run any tests.")), |
| ] |
| |
| # FIXME: Move these into json_results_generator.py |
| results_json_options = [ |
| optparse.make_option("--master-name", help="The name of the buildbot master."), |
| optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", |
| help=("The name of the builder shown on the waterfall running " |
| "this script e.g. WebKit.")), |
| optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", |
| help=("The name of the builder used in its path, e.g. " |
| "webkit-rel.")), |
| optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", |
| help=("The build number of the builder running this script.")), |
| optparse.make_option("--test-results-server", default="", |
| help=("If specified, upload results json files to this appengine " |
| "server.")), |
| ] |
| |
| option_list = (configuration_options + print_options + |
| chromium_options + results_options + test_options + |
| misc_options + results_json_options + |
| old_run_webkit_tests_compat) |
| option_parser = optparse.OptionParser(option_list=option_list) |
| |
| return option_parser.parse_args(args) |
| |
| |
| def main(): |
| options, args = parse_args() |
| port_obj = port.get(options.platform, options) |
| return run(port_obj, options, args) |
| |
| |
| if '__main__' == __name__: |
| try: |
| sys.exit(main()) |
| except KeyboardInterrupt: |
| # this mirrors what the shell normally does |
| sys.exit(signal.SIGINT + 128) |
