| #!/usr/bin/env python |
| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """A helper class for reading in and dealing with tests expectations |
| for layout tests. |
| """ |
| |
| import itertools |
| import logging |
| import re |
| |
| import webkitpy.thirdparty.simplejson as simplejson |
| |
| _log = logging.getLogger("webkitpy.layout_tests.layout_package." |
| "test_expectations") |
| |
| # Test expectation and modifier constants. |
| (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX, |
| SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16) |
| |
| # Test expectation file update action constants |
| (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) |
| |
| |
| def result_was_expected(result, expected_results, test_needs_rebaselining, |
| test_is_skipped): |
| """Returns whether we got a result we were expecting. |
| Args: |
| result: actual result of a test execution |
| expected_results: set of results listed in test_expectations |
| test_needs_rebaselining: whether test was marked as REBASELINE |
| test_is_skipped: whether test was marked as SKIP""" |
| if result in expected_results: |
| return True |
| if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results: |
| return True |
| if result == MISSING and test_needs_rebaselining: |
| return True |
| if result == SKIP and test_is_skipped: |
| return True |
| return False |
| |
| |
| def remove_pixel_failures(expected_results): |
| """Returns a copy of the expected results for a test, except that we |
| drop any pixel failures and return the remaining expectations. For example, |
| if we're not running pixel tests, then tests expected to fail as IMAGE |
| will PASS.""" |
| expected_results = expected_results.copy() |
| if IMAGE in expected_results: |
| expected_results.remove(IMAGE) |
| expected_results.add(PASS) |
| if IMAGE_PLUS_TEXT in expected_results: |
| expected_results.remove(IMAGE_PLUS_TEXT) |
| expected_results.add(TEXT) |
| return expected_results |
| |
| |
| class TestExpectations: |
| TEST_LIST = "test_expectations.txt" |
| |
| def __init__(self, port, tests, expectations, test_config, |
| is_lint_mode, overrides=None): |
| """Loads and parses the test expectations given in the string. |
| Args: |
| port: handle to object containing platform-specific functionality |
| tests: list of all of the test files |
| expectations: test expectations as a string |
| test_config: specific values to check against when |
| parsing the file (usually port.test_config(), |
| but may be different when linting or doing other things). |
| is_lint_mode: If True, just parse the expectations string |
| looking for errors. |
| overrides: test expectations that are allowed to override any |
| entries in |expectations|. This is used by callers |
| that need to manage two sets of expectations (e.g., upstream |
| and downstream expectations). |
| """ |
| self._expected_failures = TestExpectationsFile(port, expectations, |
| tests, test_config, is_lint_mode, |
| overrides=overrides) |
| |
| # TODO(ojan): Allow for removing skipped tests when getting the list of |
| # tests to run, but not when getting metrics. |
| # TODO(ojan): Replace the Get* calls here with the more sane API exposed |
| # by TestExpectationsFile below. Maybe merge the two classes entirely? |
| |
| def get_expectations_json_for_all_platforms(self): |
| return ( |
| self._expected_failures.get_expectations_json_for_all_platforms()) |
| |
| def get_rebaselining_failures(self): |
| return (self._expected_failures.get_test_set(REBASELINE, FAIL) | |
| self._expected_failures.get_test_set(REBASELINE, IMAGE) | |
| self._expected_failures.get_test_set(REBASELINE, TEXT) | |
| self._expected_failures.get_test_set(REBASELINE, |
| IMAGE_PLUS_TEXT) | |
| self._expected_failures.get_test_set(REBASELINE, AUDIO)) |
| |
| def get_options(self, test): |
| return self._expected_failures.get_options(test) |
| |
| def get_expectations(self, test): |
| return self._expected_failures.get_expectations(test) |
| |
| def get_expectations_string(self, test): |
| """Returns the expectatons for the given test as an uppercase string. |
| If there are no expectations for the test, then "PASS" is returned.""" |
| expectations = self.get_expectations(test) |
| retval = [] |
| |
| for expectation in expectations: |
| retval.append(self.expectation_to_string(expectation)) |
| |
| return " ".join(retval) |
| |
| def expectation_to_string(self, expectation): |
| """Return the uppercased string equivalent of a given expectation.""" |
| for item in TestExpectationsFile.EXPECTATIONS.items(): |
| if item[1] == expectation: |
| return item[0].upper() |
| raise ValueError(expectation) |
| |
| def get_tests_with_result_type(self, result_type): |
| return self._expected_failures.get_tests_with_result_type(result_type) |
| |
| def get_tests_with_timeline(self, timeline): |
| return self._expected_failures.get_tests_with_timeline(timeline) |
| |
| def matches_an_expected_result(self, test, result, |
| pixel_tests_are_enabled): |
| expected_results = self._expected_failures.get_expectations(test) |
| if not pixel_tests_are_enabled: |
| expected_results = remove_pixel_failures(expected_results) |
| return result_was_expected(result, expected_results, |
| self.is_rebaselining(test), self.has_modifier(test, SKIP)) |
| |
| def is_rebaselining(self, test): |
| return self._expected_failures.has_modifier(test, REBASELINE) |
| |
| def has_modifier(self, test, modifier): |
| return self._expected_failures.has_modifier(test, modifier) |
| |
| def remove_rebaselined_tests(self, tests): |
| return self._expected_failures.remove_rebaselined_tests(tests) |
| |
| |
| def strip_comments(line): |
| """Strips comments from a line and return None if the line is empty |
| or else the contents of line with leading and trailing spaces removed |
| and all other whitespace collapsed""" |
| |
| commentIndex = line.find('//') |
| if commentIndex is -1: |
| commentIndex = len(line) |
| |
| line = re.sub(r'\s+', ' ', line[:commentIndex].strip()) |
| if line == '': |
| return None |
| else: |
| return line |
| |
| |
| class ParseError(Exception): |
| def __init__(self, fatal, errors): |
| self.fatal = fatal |
| self.errors = errors |
| |
| def __str__(self): |
| return '\n'.join(map(str, self.errors)) |
| |
| def __repr__(self): |
| return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors) |
| |
| |
| class ModifiersAndExpectations: |
| """A holder for modifiers and expectations on a test that serializes to |
| JSON.""" |
| |
| def __init__(self, modifiers, expectations): |
| self.modifiers = modifiers |
| self.expectations = expectations |
| |
| |
| class ExpectationsJsonEncoder(simplejson.JSONEncoder): |
| """JSON encoder that can handle ModifiersAndExpectations objects.""" |
| def default(self, obj): |
| # A ModifiersAndExpectations object has two fields, each of which |
| # is a dict. Since JSONEncoders handle all the builtin types directly, |
| # the only time this routine should be called is on the top level |
| # object (i.e., the encoder shouldn't recurse). |
| assert isinstance(obj, ModifiersAndExpectations) |
| return {"modifiers": obj.modifiers, |
| "expectations": obj.expectations} |
| |
| |
| class TestExpectationsFile: |
| """Test expectation files consist of lines with specifications of what |
| to expect from layout test cases. The test cases can be directories |
| in which case the expectations apply to all test cases in that |
| directory and any subdirectory. The format of the file is along the |
| lines of: |
| |
| LayoutTests/fast/js/fixme.js = FAIL |
| LayoutTests/fast/js/flaky.js = FAIL PASS |
| LayoutTests/fast/js/crash.js = CRASH TIMEOUT FAIL PASS |
| ... |
| |
| To add other options: |
| SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS |
| DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS |
| DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS |
| LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS |
| LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS |
| |
| SKIP: Doesn't run the test. |
| SLOW: The test takes a long time to run, but does not timeout indefinitely. |
| WONTFIX: For tests that we never intend to pass on a given platform. |
| |
| Notes: |
| -A test cannot be both SLOW and TIMEOUT |
| -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, AUDIO, or FAIL. |
| FAIL is a legacy value that currently means either IMAGE, |
| TEXT, or IMAGE+TEXT. Once we have finished migrating the expectations, |
| we should change FAIL to have the meaning of IMAGE+TEXT and remove the |
| IMAGE+TEXT identifier. |
| -A test can be included twice, but not via the same path. |
| -If a test is included twice, then the more precise path wins. |
| -CRASH tests cannot be WONTFIX |
| """ |
| |
| EXPECTATIONS = {'pass': PASS, |
| 'fail': FAIL, |
| 'text': TEXT, |
| 'image': IMAGE, |
| 'image+text': IMAGE_PLUS_TEXT, |
| 'audio': AUDIO, |
| 'timeout': TIMEOUT, |
| 'crash': CRASH, |
| 'missing': MISSING} |
| |
| EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'), |
| PASS: ('pass', 'passes'), |
| FAIL: ('failure', 'failures'), |
| TEXT: ('text diff mismatch', |
| 'text diff mismatch'), |
| IMAGE: ('image mismatch', 'image mismatch'), |
| IMAGE_PLUS_TEXT: ('image and text mismatch', |
| 'image and text mismatch'), |
| AUDIO: ('audio mismatch', 'audio mismatch'), |
| CRASH: ('DumpRenderTree crash', |
| 'DumpRenderTree crashes'), |
| TIMEOUT: ('test timed out', 'tests timed out'), |
| MISSING: ('no expected result found', |
| 'no expected results found')} |
| |
| EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT, |
| TEXT, IMAGE, AUDIO, FAIL, SKIP) |
| |
| BUILD_TYPES = ('debug', 'release') |
| |
| MODIFIERS = {'skip': SKIP, |
| 'wontfix': WONTFIX, |
| 'slow': SLOW, |
| 'rebaseline': REBASELINE, |
| 'none': NONE} |
| |
| TIMELINES = {'wontfix': WONTFIX, |
| 'now': NOW} |
| |
| RESULT_TYPES = {'skip': SKIP, |
| 'pass': PASS, |
| 'fail': FAIL, |
| 'flaky': FLAKY} |
| |
| def __init__(self, port, expectations, full_test_list, |
| test_config, is_lint_mode, overrides=None): |
| # See argument documentation in TestExpectation(), above. |
| |
| self._port = port |
| self._fs = port._filesystem |
| self._expectations = expectations |
| self._full_test_list = full_test_list |
| self._test_config = test_config |
| self._is_lint_mode = is_lint_mode |
| self._overrides = overrides |
| self._errors = [] |
| self._non_fatal_errors = [] |
| |
| # Maps relative test paths as listed in the expectations file to a |
| # list of maps containing modifiers and expectations for each time |
| # the test is listed in the expectations file. We use this to |
| # keep a representation of the entire list of expectations, even |
| # invalid ones. |
| self._all_expectations = {} |
| |
| # Maps a test to its list of expectations. |
| self._test_to_expectations = {} |
| |
| # Maps a test to its list of options (string values) |
| self._test_to_options = {} |
| |
| # Maps a test to its list of modifiers: the constants associated with |
| # the options minus any bug or platform strings |
| self._test_to_modifiers = {} |
| |
| # Maps a test to the base path that it was listed with in the list and |
| # the number of matches that base path had. |
| self._test_list_paths = {} |
| |
| self._modifier_to_tests = self._dict_of_sets(self.MODIFIERS) |
| self._expectation_to_tests = self._dict_of_sets(self.EXPECTATIONS) |
| self._timeline_to_tests = self._dict_of_sets(self.TIMELINES) |
| self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES) |
| |
| self._read(self._get_iterable_expectations(self._expectations), |
| overrides_allowed=False) |
| |
| # List of tests that are in the overrides file (used for checking for |
| # duplicates inside the overrides file itself). Note that just because |
| # a test is in this set doesn't mean it's necessarily overridding a |
| # expectation in the regular expectations; the test might not be |
| # mentioned in the regular expectations file at all. |
| self._overridding_tests = set() |
| |
| if overrides: |
| self._read(self._get_iterable_expectations(self._overrides), |
| overrides_allowed=True) |
| |
| self._handle_any_read_errors() |
| self._process_tests_without_expectations() |
| |
| def _handle_any_read_errors(self): |
| if len(self._errors) or len(self._non_fatal_errors): |
| _log.error("FAILURES FOR %s" % str(self._test_config)) |
| |
| for error in self._errors: |
| _log.error(error) |
| for error in self._non_fatal_errors: |
| _log.error(error) |
| |
| if len(self._errors): |
| raise ParseError(fatal=True, errors=self._errors) |
| if len(self._non_fatal_errors) and self._is_lint_mode: |
| raise ParseError(fatal=False, errors=self._non_fatal_errors) |
| |
| def _process_tests_without_expectations(self): |
| expectations = set([PASS]) |
| options = [] |
| modifiers = [] |
| num_matches = 0 |
| if self._full_test_list: |
| for test in self._full_test_list: |
| if not test in self._test_list_paths: |
| self._add_test(test, modifiers, num_matches, expectations, |
| options, overrides_allowed=False) |
| |
| def _dict_of_sets(self, strings_to_constants): |
| """Takes a dict of strings->constants and returns a dict mapping |
| each constant to an empty set.""" |
| d = {} |
| for c in strings_to_constants.values(): |
| d[c] = set() |
| return d |
| |
| def _get_iterable_expectations(self, expectations_str): |
| """Returns an object that can be iterated over. Allows for not caring |
| about whether we're iterating over a file or a new-line separated |
| string.""" |
| iterable = [x + "\n" for x in expectations_str.split("\n")] |
| # Strip final entry if it's empty to avoid added in an extra |
| # newline. |
| if iterable[-1] == "\n": |
| return iterable[:-1] |
| return iterable |
| |
| def get_test_set(self, modifier, expectation=None, include_skips=True): |
| if expectation is None: |
| tests = self._modifier_to_tests[modifier] |
| else: |
| tests = (self._expectation_to_tests[expectation] & |
| self._modifier_to_tests[modifier]) |
| |
| if not include_skips: |
| tests = tests - self.get_test_set(SKIP, expectation) |
| |
| return tests |
| |
| def get_tests_with_result_type(self, result_type): |
| return self._result_type_to_tests[result_type] |
| |
| def get_tests_with_timeline(self, timeline): |
| return self._timeline_to_tests[timeline] |
| |
| def get_options(self, test): |
| """This returns the entire set of options for the given test |
| (the modifiers plus the BUGXXXX identifier). This is used by the |
| LTTF dashboard.""" |
| return self._test_to_options[test] |
| |
| def has_modifier(self, test, modifier): |
| return test in self._modifier_to_tests[modifier] |
| |
| def get_expectations(self, test): |
| return self._test_to_expectations[test] |
| |
| def get_expectations_json_for_all_platforms(self): |
| # Specify separators in order to get compact encoding. |
| return ExpectationsJsonEncoder(separators=(',', ':')).encode( |
| self._all_expectations) |
| |
| def get_non_fatal_errors(self): |
| return self._non_fatal_errors |
| |
| def remove_rebaselined_tests(self, tests): |
| """Returns a copy of the expectations with the tests removed.""" |
| lines = [] |
| for (lineno, line) in enumerate(self._get_iterable_expectations(self._expectations)): |
| test, options, _ = self.parse_expectations_line(line, lineno) |
| if not (test and test in tests and 'rebaseline' in options): |
| lines.append(line) |
| return ''.join(lines) |
| |
| def parse_expectations_line(self, line, lineno): |
| """Parses a line from test_expectations.txt and returns a tuple |
| with the test path, options as a list, expectations as a list.""" |
| line = strip_comments(line) |
| if not line: |
| return (None, None, None) |
| |
| options = [] |
| if line.find(":") is -1: |
| self._add_error(lineno, "Missing a ':'", line) |
| return (None, None, None) |
| |
| parts = line.split(':') |
| |
| # FIXME: verify that there is exactly one colon in the line. |
| |
| options = self._get_options_list(parts[0]) |
| test_and_expectation = parts[1].split('=') |
| test = test_and_expectation[0].strip() |
| if (len(test_and_expectation) is not 2): |
| self._add_error(lineno, "Missing expectations.", |
| test_and_expectation) |
| expectations = None |
| else: |
| expectations = self._get_options_list(test_and_expectation[1]) |
| |
| return (test, options, expectations) |
| |
| def _add_to_all_expectations(self, test, options, expectations): |
| # Make all paths unix-style so the dashboard doesn't need to. |
| test = test.replace('\\', '/') |
| if not test in self._all_expectations: |
| self._all_expectations[test] = [] |
| self._all_expectations[test].append( |
| ModifiersAndExpectations(options, expectations)) |
| |
| def _read(self, expectations, overrides_allowed): |
| """For each test in an expectations iterable, generate the |
| expectations for it.""" |
| lineno = 0 |
| matcher = ModifierMatcher(self._test_config) |
| for line in expectations: |
| lineno += 1 |
| self._process_line(line, lineno, matcher, overrides_allowed) |
| |
| def _process_line(self, line, lineno, matcher, overrides_allowed): |
| test_list_path, options, expectations = \ |
| self.parse_expectations_line(line, lineno) |
| if not expectations: |
| return |
| |
| self._add_to_all_expectations(test_list_path, |
| " ".join(options).upper(), |
| " ".join(expectations).upper()) |
| |
| num_matches = self._check_options(matcher, options, lineno, |
| test_list_path) |
| if num_matches == ModifierMatcher.NO_MATCH: |
| return |
| |
| expectations = self._parse_expectations(expectations, lineno, |
| test_list_path) |
| |
| self._check_options_against_expectations(options, expectations, |
| lineno, test_list_path) |
| |
| if self._check_path_does_not_exist(lineno, test_list_path): |
| return |
| |
| if not self._full_test_list: |
| tests = [test_list_path] |
| else: |
| tests = self._expand_tests(test_list_path) |
| |
| modifiers = [o for o in options if o in self.MODIFIERS] |
| self._add_tests(tests, expectations, test_list_path, lineno, |
| modifiers, num_matches, options, overrides_allowed) |
| |
| def _get_options_list(self, listString): |
| return [part.strip().lower() for part in listString.strip().split(' ')] |
| |
| def _parse_expectations(self, expectations, lineno, test_list_path): |
| result = set() |
| for part in expectations: |
| if not part in self.EXPECTATIONS: |
| self._add_error(lineno, 'Unsupported expectation: %s' % part, |
| test_list_path) |
| continue |
| expectation = self.EXPECTATIONS[part] |
| result.add(expectation) |
| return result |
| |
| def _check_options(self, matcher, options, lineno, test_list_path): |
| match_result = self._check_syntax(matcher, options, lineno, |
| test_list_path) |
| self._check_semantics(options, lineno, test_list_path) |
| return match_result.num_matches |
| |
| def _check_syntax(self, matcher, options, lineno, test_list_path): |
| match_result = matcher.match(options) |
| for error in match_result.errors: |
| self._add_error(lineno, error, test_list_path) |
| for warning in match_result.warnings: |
| self._log_non_fatal_error(lineno, warning, test_list_path) |
| return match_result |
| |
| def _check_semantics(self, options, lineno, test_list_path): |
| has_wontfix = 'wontfix' in options |
| has_bug = False |
| for opt in options: |
| if opt.startswith('bug'): |
| has_bug = True |
| if re.match('bug\d+', opt): |
| self._add_error(lineno, |
| 'BUG\d+ is not allowed, must be one of ' |
| 'BUGCR\d+, BUGWK\d+, BUGV8_\d+, ' |
| 'or a non-numeric bug identifier.', test_list_path) |
| |
| if not has_bug and not has_wontfix: |
| self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.', |
| test_list_path) |
| |
| if self._is_lint_mode and 'rebaseline' in options: |
| self._add_error(lineno, |
| 'REBASELINE should only be used for running rebaseline.py. ' |
| 'Cannot be checked in.', test_list_path) |
| |
| def _check_options_against_expectations(self, options, expectations, |
| lineno, test_list_path): |
| if 'slow' in options and TIMEOUT in expectations: |
| self._add_error(lineno, |
| 'A test can not be both SLOW and TIMEOUT. If it times out ' |
| 'indefinitely, then it should be just TIMEOUT.', test_list_path) |
| |
| def _check_path_does_not_exist(self, lineno, test_list_path): |
| full_path = self._fs.join(self._port.layout_tests_dir(), |
| test_list_path) |
| full_path = self._fs.normpath(full_path) |
| # WebKit's way of skipping tests is to add a -disabled suffix. |
| # So we should consider the path existing if the path or the |
| # -disabled version exists. |
| if (not self._port.path_exists(full_path) |
| and not self._port.path_exists(full_path + '-disabled')): |
| # Log a non fatal error here since you hit this case any |
| # time you update test_expectations.txt without syncing |
| # the LayoutTests directory |
| self._log_non_fatal_error(lineno, 'Path does not exist.', |
| test_list_path) |
| return True |
| return False |
| |
| def _expand_tests(self, test_list_path): |
| """Convert the test specification to an absolute, normalized |
| path and make sure directories end with the OS path separator.""" |
| # FIXME: full_test_list can quickly contain a big amount of |
| # elements. We should consider at some point to use a more |
| # efficient structure instead of a list. Maybe a dictionary of |
| # lists to represent the tree of tests, leaves being test |
| # files and nodes being categories. |
| |
| path = self._fs.join(self._port.layout_tests_dir(), test_list_path) |
| path = self._fs.normpath(path) |
| if self._fs.isdir(path): |
| # this is a test category, return all the tests of the category. |
| path = self._fs.join(path, '') |
| |
| return [test for test in self._full_test_list if test.startswith(path)] |
| |
| # this is a test file, do a quick check if it's in the |
| # full test suite. |
| result = [] |
| if path in self._full_test_list: |
| result = [path, ] |
| return result |
| |
| def _add_tests(self, tests, expectations, test_list_path, lineno, |
| modifiers, num_matches, options, overrides_allowed): |
| for test in tests: |
| if self._already_seen_better_match(test, test_list_path, |
| num_matches, lineno, overrides_allowed): |
| continue |
| |
| self._clear_expectations_for_test(test, test_list_path) |
| self._test_list_paths[test] = (self._fs.normpath(test_list_path), |
| num_matches, lineno) |
| self._add_test(test, modifiers, num_matches, expectations, options, |
| overrides_allowed) |
| |
| def _add_test(self, test, modifiers, num_matches, expectations, options, |
| overrides_allowed): |
| """Sets the expected state for a given test. |
| |
| This routine assumes the test has not been added before. If it has, |
| use _clear_expectations_for_test() to reset the state prior to |
| calling this. |
| |
| Args: |
| test: test to add |
| modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.) |
| num_matches: number of modifiers that matched the configuration |
| expectations: sequence of expectations (PASS, IMAGE, etc.) |
| options: sequence of keywords and bug identifiers. |
| overrides_allowed: whether we're parsing the regular expectations |
| or the overridding expectations""" |
| self._test_to_expectations[test] = expectations |
| for expectation in expectations: |
| self._expectation_to_tests[expectation].add(test) |
| |
| self._test_to_options[test] = options |
| self._test_to_modifiers[test] = set() |
| for modifier in modifiers: |
| mod_value = self.MODIFIERS[modifier] |
| self._modifier_to_tests[mod_value].add(test) |
| self._test_to_modifiers[test].add(mod_value) |
| |
| if 'wontfix' in modifiers: |
| self._timeline_to_tests[WONTFIX].add(test) |
| else: |
| self._timeline_to_tests[NOW].add(test) |
| |
| if 'skip' in modifiers: |
| self._result_type_to_tests[SKIP].add(test) |
| elif expectations == set([PASS]): |
| self._result_type_to_tests[PASS].add(test) |
| elif len(expectations) > 1: |
| self._result_type_to_tests[FLAKY].add(test) |
| else: |
| self._result_type_to_tests[FAIL].add(test) |
| |
| if overrides_allowed: |
| self._overridding_tests.add(test) |
| |
| def _clear_expectations_for_test(self, test, test_list_path): |
| """Remove prexisting expectations for this test. |
| This happens if we are seeing a more precise path |
| than a previous listing. |
| """ |
| if test in self._test_list_paths: |
| self._test_to_expectations.pop(test, '') |
| self._remove_from_sets(test, self._expectation_to_tests) |
| self._remove_from_sets(test, self._modifier_to_tests) |
| self._remove_from_sets(test, self._timeline_to_tests) |
| self._remove_from_sets(test, self._result_type_to_tests) |
| |
| self._test_list_paths[test] = self._fs.normpath(test_list_path) |
| |
| def _remove_from_sets(self, test, dict): |
| """Removes the given test from the sets in the dictionary. |
| |
| Args: |
| test: test to look for |
| dict: dict of sets of files""" |
| for set_of_tests in dict.itervalues(): |
| if test in set_of_tests: |
| set_of_tests.remove(test) |
| |
| def _already_seen_better_match(self, test, test_list_path, num_matches, |
| lineno, overrides_allowed): |
| """Returns whether we've seen a better match already in the file. |
| |
| Returns True if we've already seen a test_list_path that matches more of the test |
| than this path does |
| """ |
| # FIXME: See comment below about matching test configs and num_matches. |
| |
| if not test in self._test_list_paths: |
| # We've never seen this test before. |
| return False |
| |
| prev_base_path, prev_num_matches, prev_lineno = self._test_list_paths[test] |
| base_path = self._fs.normpath(test_list_path) |
| |
| if len(prev_base_path) > len(base_path): |
| # The previous path matched more of the test. |
| return True |
| |
| if len(prev_base_path) < len(base_path): |
| # This path matches more of the test. |
| return False |
| |
| if overrides_allowed and test not in self._overridding_tests: |
| # We have seen this path, but that's okay because it is |
| # in the overrides and the earlier path was in the |
| # expectations (not the overrides). |
| return False |
| |
| # At this point we know we have seen a previous exact match on this |
| # base path, so we need to check the two sets of modifiers. |
| |
| if overrides_allowed: |
| expectation_source = "override" |
| else: |
| expectation_source = "expectation" |
| |
| # FIXME: This code was originally designed to allow lines that matched |
| # more modifiers to override lines that matched fewer modifiers. |
| # However, we currently view these as errors. If we decide to make |
| # this policy permanent, we can probably simplify this code |
| # and the ModifierMatcher code a fair amount. |
| # |
| # To use the "more modifiers wins" policy, change the "_add_error" lines for overrides |
| # to _log_non_fatal_error() and change the commented-out "return False". |
| |
| if prev_num_matches == num_matches: |
| self._add_error(lineno, |
| 'Duplicate or ambiguous %s.' % expectation_source, |
| test) |
| return True |
| |
| if prev_num_matches < num_matches: |
| self._add_error(lineno, |
| 'More specific entry on line %d overrides line %d' % |
| (lineno, prev_lineno), test_list_path) |
| # FIXME: return False if we want more specific to win. |
| return True |
| |
| self._add_error(lineno, |
| 'More specific entry on line %d overrides line %d' % |
| (prev_lineno, lineno), test_list_path) |
| return True |
| |
| def _add_error(self, lineno, msg, path): |
| """Reports an error that will prevent running the tests. Does not |
| immediately raise an exception because we'd like to aggregate all the |
| errors so they can all be printed out.""" |
| self._errors.append('Line:%s %s %s' % (lineno, msg, path)) |
| |
| def _log_non_fatal_error(self, lineno, msg, path): |
| """Reports an error that will not prevent running the tests. These are |
| still errors, but not bad enough to warrant breaking test running.""" |
| self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path)) |
| |
| |
| class ModifierMatchResult(object): |
| def __init__(self, options): |
| self.num_matches = ModifierMatcher.NO_MATCH |
| self.options = options |
| self.errors = [] |
| self.warnings = [] |
| self.modifiers = [] |
| self._matched_regexes = set() |
| self._matched_macros = set() |
| |
| |
| class ModifierMatcher(object): |
| |
| """ |
| This class manages the interpretation of the "modifiers" for a given |
| line in the expectations file. Modifiers are the tokens that appear to the |
| left of the colon on a line. For example, "BUG1234", "DEBUG", and "WIN" are |
| all modifiers. This class gets what the valid modifiers are, and which |
| modifiers are allowed to exist together on a line, from the |
| TestConfiguration object that is passed in to the call. |
| |
| This class detects *intra*-line errors like unknown modifiers, but |
| does not detect *inter*-line modifiers like duplicate expectations. |
| |
| More importantly, this class is also used to determine if a given line |
| matches the port in question. Matches are ranked according to the number |
| of modifiers that match on a line. A line with no modifiers matches |
| everything and has a score of zero. A line with one modifier matches only |
| ports that have that modifier and gets a score of 1, and so one. Ports |
| that don't match at all get a score of -1. |
| |
| Given two lines in a file that apply to the same test, if both expectations |
| match the current config, then the expectation is considered ambiguous, |
| even if one expectation matches more of the config than the other. For |
| example, in: |
| |
| BUG1 RELEASE : foo.html = FAIL |
| BUG1 WIN RELEASE : foo.html = PASS |
| BUG2 WIN : bar.html = FAIL |
| BUG2 DEBUG : bar.html = PASS |
| |
| lines 1 and 2 would produce an error on a Win XP Release bot (the scores |
| would be 1 and 2, respectively), and lines three and four would produce |
| a duplicate expectation on a Win Debug bot since both the 'win' and the |
| 'debug' expectations would apply (both had scores of 1). |
| |
| In addition to the definitions of all of the modifiers, the class |
| supports "macros" that are expanded prior to interpretation, and "ignore |
| regexes" that can be used to skip over modifiers like the BUG* modifiers. |
| """ |
| MACROS = { |
| 'mac-snowleopard': ['mac', 'snowleopard'], |
| 'mac-leopard': ['mac', 'leopard'], |
| 'win-xp': ['win', 'xp'], |
| 'win-vista': ['win', 'vista'], |
| 'win-win7': ['win', 'win7'], |
| } |
| |
| # We don't include the "none" modifier because it isn't actually legal. |
| REGEXES_TO_IGNORE = (['bug\w+'] + |
| TestExpectationsFile.MODIFIERS.keys()[:-1]) |
| DUPLICATE_REGEXES_ALLOWED = ['bug\w+'] |
| |
| # Magic value returned when the options don't match. |
| NO_MATCH = -1 |
| |
| # FIXME: The code currently doesn't detect combinations of modifiers |
| # that are syntactically valid but semantically invalid, like |
| # 'MAC XP'. See ModifierMatchTest.test_invalid_combinations() in the |
| # _unittest.py file. |
| |
| def __init__(self, test_config): |
| """Initialize a ModifierMatcher argument with the TestConfiguration it |
| should be matched against.""" |
| self.test_config = test_config |
| self.allowed_configurations = test_config.all_test_configurations() |
| self.macros = self.MACROS |
| |
| self.regexes_to_ignore = {} |
| for regex_str in self.REGEXES_TO_IGNORE: |
| self.regexes_to_ignore[regex_str] = re.compile(regex_str) |
| |
| # Keep a set of all of the legal modifiers for quick checking. |
| self._all_modifiers = set() |
| |
| # Keep a dict mapping values back to their categories. |
| self._categories_for_modifiers = {} |
| for config in self.allowed_configurations: |
| for category, modifier in config.items(): |
| self._categories_for_modifiers[modifier] = category |
| self._all_modifiers.add(modifier) |
| |
| def match(self, options): |
| """Checks a list of options against the config set in the constructor. |
| Options may be either actual modifier strings, "macro" strings |
| that get expanded to a list of modifiers, or strings that are allowed |
| to be ignored. All of the options must be passed in in lower case. |
| |
| Returns the number of matching categories, or NO_MATCH (-1) if it |
| doesn't match or there were errors found. Matches are prioritized |
| by the number of matching categories, because the more specific |
| the options list, the more categories will match. |
| |
| The results of the most recent match are available in the 'options', |
| 'modifiers', 'num_matches', 'errors', and 'warnings' properties. |
| """ |
| result = ModifierMatchResult(options) |
| self._parse(result) |
| if result.errors: |
| return result |
| self._count_matches(result) |
| return result |
| |
| def _parse(self, result): |
| # FIXME: Should we warn about lines having every value in a category? |
| for option in result.options: |
| self._parse_one(option, result) |
| |
| def _parse_one(self, option, result): |
| if option in self._all_modifiers: |
| self._add_modifier(option, result) |
| elif option in self.macros: |
| self._expand_macro(option, result) |
| elif not self._matches_any_regex(option, result): |
| result.errors.append("Unrecognized option '%s'" % option) |
| |
| def _add_modifier(self, option, result): |
| if option in result.modifiers: |
| result.errors.append("More than one '%s'" % option) |
| else: |
| result.modifiers.append(option) |
| |
| def _expand_macro(self, macro, result): |
| if macro in result._matched_macros: |
| result.errors.append("More than one '%s'" % macro) |
| return |
| |
| mods = [] |
| for modifier in self.macros[macro]: |
| if modifier in result.options: |
| result.errors.append("Can't specify both modifier '%s' and " |
| "macro '%s'" % (modifier, macro)) |
| else: |
| mods.append(modifier) |
| result._matched_macros.add(macro) |
| result.modifiers.extend(mods) |
| |
| def _matches_any_regex(self, option, result): |
| for regex_str, pattern in self.regexes_to_ignore.iteritems(): |
| if pattern.match(option): |
| self._handle_regex_match(regex_str, result) |
| return True |
| return False |
| |
| def _handle_regex_match(self, regex_str, result): |
| if (regex_str in result._matched_regexes and |
| regex_str not in self.DUPLICATE_REGEXES_ALLOWED): |
| result.errors.append("More than one option matching '%s'" % |
| regex_str) |
| else: |
| result._matched_regexes.add(regex_str) |
| |
| def _count_matches(self, result): |
| """Returns the number of modifiers that match the test config.""" |
| categorized_modifiers = self._group_by_category(result.modifiers) |
| result.num_matches = 0 |
| for category, modifier in self.test_config.items(): |
| if category in categorized_modifiers: |
| if modifier in categorized_modifiers[category]: |
| result.num_matches += 1 |
| else: |
| result.num_matches = self.NO_MATCH |
| return |
| |
| def _group_by_category(self, modifiers): |
| # Returns a dict of category name -> list of modifiers. |
| modifiers_by_category = {} |
| for m in modifiers: |
| modifiers_by_category.setdefault(self._category(m), []).append(m) |
| return modifiers_by_category |
| |
| def _category(self, modifier): |
| return self._categories_for_modifiers[modifier] |
