| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| from datetime import datetime |
| from django.utils import simplejson |
| import logging |
| |
| from model.testfile import TestFile |
| |
| JSON_RESULTS_FILE = "results.json" |
| JSON_RESULTS_FILE_SMALL = "results-small.json" |
| JSON_RESULTS_PREFIX = "ADD_RESULTS(" |
| JSON_RESULTS_SUFFIX = ");" |
| JSON_RESULTS_VERSION_KEY = "version" |
| JSON_RESULTS_BUILD_NUMBERS = "buildNumbers" |
| JSON_RESULTS_TESTS = "tests" |
| JSON_RESULTS_RESULTS = "results" |
| JSON_RESULTS_TIMES = "times" |
| JSON_RESULTS_PASS = "P" |
| JSON_RESULTS_NO_DATA = "N" |
| JSON_RESULTS_MIN_TIME = 1 |
| JSON_RESULTS_VERSION = 3 |
| JSON_RESULTS_MAX_BUILDS = 750 |
| JSON_RESULTS_MAX_BUILDS_SMALL = 200 |
| |
| |
| class JsonResults(object): |
| @classmethod |
| def _strip_prefix_suffix(cls, data): |
| """Strip out prefix and suffix of json results string. |
| |
| Args: |
| data: json file content. |
| |
| Returns: |
| json string without prefix and suffix. |
| """ |
| |
| assert(data.startswith(JSON_RESULTS_PREFIX)) |
| assert(data.endswith(JSON_RESULTS_SUFFIX)) |
| |
| return data[len(JSON_RESULTS_PREFIX): |
| len(data) - len(JSON_RESULTS_SUFFIX)] |
| |
| @classmethod |
| def _generate_file_data(cls, json, sort_keys=False): |
| """Given json string, generate file content data by adding |
| prefix and suffix. |
| |
| Args: |
| json: json string without prefix and suffix. |
| |
| Returns: |
| json file data. |
| """ |
| |
| data = simplejson.dumps(json, separators=(',', ':'), |
| sort_keys=sort_keys) |
| return JSON_RESULTS_PREFIX + data + JSON_RESULTS_SUFFIX |
| |
| @classmethod |
| def _load_json(cls, file_data): |
| """Load json file to a python object. |
| |
| Args: |
| file_data: json file content. |
| |
| Returns: |
| json object or |
| None on failure. |
| """ |
| |
| json_results_str = cls._strip_prefix_suffix(file_data) |
| if not json_results_str: |
| logging.warning("No json results data.") |
| return None |
| |
| try: |
| return simplejson.loads(json_results_str) |
| except Exception, err: |
| logging.debug(json_results_str) |
| logging.error("Failed to load json results: %s", str(err)) |
| return None |
| |
| @classmethod |
| def _merge_json(cls, aggregated_json, incremental_json, num_runs): |
| """Merge incremental json into aggregated json results. |
| |
| Args: |
| aggregated_json: aggregated json object. |
| incremental_json: incremental json object. |
| num_runs: number of total runs to include. |
| |
| Returns: |
| True if merge succeeds or |
| False on failure. |
| """ |
| |
| # Merge non tests property data. |
| # Tests properties are merged in _merge_tests. |
| if not cls._merge_non_test_data(aggregated_json, incremental_json, num_runs): |
| return False |
| |
| # Merge tests results and times |
| incremental_tests = incremental_json[JSON_RESULTS_TESTS] |
| if incremental_tests: |
| aggregated_tests = aggregated_json[JSON_RESULTS_TESTS] |
| cls._merge_tests(aggregated_tests, incremental_tests, num_runs) |
| |
| return True |
| |
| @classmethod |
| def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs): |
| """Merge incremental non tests property data into aggregated json results. |
| |
| Args: |
| aggregated_json: aggregated json object. |
| incremental_json: incremental json object. |
| num_runs: number of total runs to include. |
| |
| Returns: |
| True if merge succeeds or |
| False on failure. |
| """ |
| |
| incremental_builds = incremental_json[JSON_RESULTS_BUILD_NUMBERS] |
| aggregated_builds = aggregated_json[JSON_RESULTS_BUILD_NUMBERS] |
| aggregated_build_number = int(aggregated_builds[0]) |
| # Loop through all incremental builds, start from the oldest run. |
| for index in reversed(range(len(incremental_builds))): |
| build_number = int(incremental_builds[index]) |
| logging.debug("Merging build %s, incremental json index: %d.", |
| build_number, index) |
| |
| # Return if not all build numbers in the incremental json results |
| # are newer than the most recent build in the aggregated results. |
| # FIXME: make this case work. |
| if build_number < aggregated_build_number: |
| logging.warning(("Build %d in incremental json is older than " |
| "the most recent build in aggregated results: %d"), |
| build_number, aggregated_build_number) |
| return False |
| |
| # Return if the build number is duplicated. |
| # FIXME: skip the duplicated build and merge rest of the results. |
| # Need to be careful on skiping the corresponding value in |
| # _merge_tests because the property data for each test could |
| # be accumulated. |
| if build_number == aggregated_build_number: |
| logging.warning("Duplicate build %d in incremental json", |
| build_number) |
| return False |
| |
| # Merge this build into aggreagated results. |
| cls._merge_one_build(aggregated_json, incremental_json, index, num_runs) |
| |
| return True |
| |
| @classmethod |
| def _merge_one_build(cls, aggregated_json, incremental_json, |
| incremental_index, num_runs): |
| """Merge one build of incremental json into aggregated json results. |
| |
| Args: |
| aggregated_json: aggregated json object. |
| incremental_json: incremental json object. |
| incremental_index: index of the incremental json results to merge. |
| num_runs: number of total runs to include. |
| """ |
| |
| for key in incremental_json.keys(): |
| # Merge json results except "tests" properties (results, times etc). |
| # "tests" properties will be handled separately. |
| if key == JSON_RESULTS_TESTS: |
| continue |
| |
| if key in aggregated_json: |
| aggregated_json[key].insert( |
| 0, incremental_json[key][incremental_index]) |
| aggregated_json[key] = \ |
| aggregated_json[key][:num_runs] |
| else: |
| aggregated_json[key] = incremental_json[key] |
| |
| @classmethod |
| def _merge_tests(cls, aggregated_json, incremental_json, num_runs): |
| """Merge "tests" properties:results, times. |
| |
| Args: |
| aggregated_json: aggregated json object. |
| incremental_json: incremental json object. |
| num_runs: number of total runs to include. |
| """ |
| |
| all_tests = (set(aggregated_json.iterkeys()) | |
| set(incremental_json.iterkeys())) |
| for test_name in all_tests: |
| if test_name in aggregated_json: |
| aggregated_test = aggregated_json[test_name] |
| if test_name in incremental_json: |
| incremental_test = incremental_json[test_name] |
| results = incremental_test[JSON_RESULTS_RESULTS] |
| times = incremental_test[JSON_RESULTS_TIMES] |
| else: |
| results = [[1, JSON_RESULTS_NO_DATA]] |
| times = [[1, 0]] |
| |
| cls._insert_item_run_length_encoded( |
| results, aggregated_test[JSON_RESULTS_RESULTS], num_runs) |
| cls._insert_item_run_length_encoded( |
| times, aggregated_test[JSON_RESULTS_TIMES], num_runs) |
| cls._normalize_results_json(test_name, aggregated_json, num_runs) |
| else: |
| aggregated_json[test_name] = incremental_json[test_name] |
| |
| @classmethod |
| def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs): |
| """Inserts the incremental run-length encoded results into the aggregated |
| run-length encoded results. |
| |
| Args: |
| incremental_item: incremental run-length encoded results. |
| aggregated_item: aggregated run-length encoded results. |
| num_runs: number of total runs to include. |
| """ |
| |
| for item in incremental_item: |
| if len(aggregated_item) and item[1] == aggregated_item[0][1]: |
| aggregated_item[0][0] = min( |
| aggregated_item[0][0] + item[0], num_runs) |
| else: |
| aggregated_item.insert(0, item) |
| |
| @classmethod |
| def _normalize_results_json(cls, test_name, aggregated_json, num_runs): |
| """ Prune tests where all runs pass or tests that no longer exist and |
| truncate all results to num_runs. |
| |
| Args: |
| test_name: Name of the test. |
| aggregated_json: The JSON object with all the test results for |
| this builder. |
| num_runs: number of total runs to include. |
| """ |
| |
| aggregated_test = aggregated_json[test_name] |
| aggregated_test[JSON_RESULTS_RESULTS] = \ |
| cls._remove_items_over_max_number_of_builds( |
| aggregated_test[JSON_RESULTS_RESULTS], num_runs) |
| aggregated_test[JSON_RESULTS_TIMES] = \ |
| cls._remove_items_over_max_number_of_builds( |
| aggregated_test[JSON_RESULTS_TIMES], num_runs) |
| |
| is_all_pass = cls._is_results_all_of_type( |
| aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_PASS) |
| is_all_no_data = cls._is_results_all_of_type( |
| aggregated_test[JSON_RESULTS_RESULTS], JSON_RESULTS_NO_DATA) |
| |
| max_time = max( |
| [time[1] for time in aggregated_test[JSON_RESULTS_TIMES]]) |
| # Remove all passes/no-data from the results to reduce noise and |
| # filesize. If a test passes every run, but |
| # takes >= JSON_RESULTS_MIN_TIME to run, don't throw away the data. |
| if (is_all_no_data or |
| (is_all_pass and max_time < JSON_RESULTS_MIN_TIME)): |
| del aggregated_json[test_name] |
| |
| @classmethod |
| def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs): |
| """Removes items from the run-length encoded list after the final |
| item that exceeds the max number of builds to track. |
| |
| Args: |
| encoded_results: run-length encoded results. An array of arrays, e.g. |
| [[3,'A'],[1,'Q']] encodes AAAQ. |
| num_runs: number of total runs to include. |
| """ |
| num_builds = 0 |
| index = 0 |
| for result in encoded_list: |
| num_builds = num_builds + result[0] |
| index = index + 1 |
| if num_builds >= num_runs: |
| return encoded_list[:index] |
| |
| return encoded_list |
| |
| @classmethod |
| def _is_results_all_of_type(cls, results, type): |
| """Returns whether all the results are of the given type |
| (e.g. all passes). |
| """ |
| |
| return len(results) == 1 and results[0][1] == type |
| |
| @classmethod |
| def _check_json(cls, builder, json): |
| """Check whether the given json is valid. |
| |
| Args: |
| builder: builder name this json is for. |
| json: json object to check. |
| |
| Returns: |
| True if the json is valid or |
| False otherwise. |
| """ |
| |
| version = json[JSON_RESULTS_VERSION_KEY] |
| if version > JSON_RESULTS_VERSION: |
| logging.error("Results JSON version '%s' is not supported.", |
| version) |
| return False |
| |
| if not builder in json: |
| logging.error("Builder '%s' is not in json results.", builder) |
| return False |
| |
| results_for_builder = json[builder] |
| if not JSON_RESULTS_BUILD_NUMBERS in results_for_builder: |
| logging.error("Missing build number in json results.") |
| return False |
| |
| return True |
| |
| @classmethod |
| def merge(cls, builder, aggregated, incremental, num_runs, sort_keys=False): |
| """Merge incremental json file data with aggregated json file data. |
| |
| Args: |
| builder: builder name. |
| aggregated: aggregated json file data. |
| incremental: incremental json file data. |
| sort_key: whether or not to sort key when dumping json results. |
| |
| Returns: |
| Merged json file data if merge succeeds or |
| None on failure. |
| """ |
| |
| if not incremental: |
| logging.warning("Nothing to merge.") |
| return None |
| |
| logging.info("Loading incremental json...") |
| incremental_json = cls._load_json(incremental) |
| if not incremental_json: |
| return None |
| |
| logging.info("Checking incremental json...") |
| if not cls._check_json(builder, incremental_json): |
| return None |
| |
| logging.info("Loading existing aggregated json...") |
| aggregated_json = cls._load_json(aggregated) |
| if not aggregated_json: |
| return incremental |
| |
| logging.info("Checking existing aggregated json...") |
| if not cls._check_json(builder, aggregated_json): |
| return incremental |
| |
| logging.info("Merging json results...") |
| try: |
| if not cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs): |
| return None |
| except Exception, err: |
| logging.error("Failed to merge json results: %s", str(err)) |
| return None |
| |
| aggregated_json[JSON_RESULTS_VERSION_KEY] = JSON_RESULTS_VERSION |
| |
| return cls._generate_file_data(aggregated_json, sort_keys) |
| |
| @classmethod |
| def update(cls, master, builder, test_type, incremental): |
| """Update datastore json file data by merging it with incremental json |
| file. Writes the large file and a small file. The small file just stores |
| fewer runs. |
| |
| Args: |
| master: master name. |
| builder: builder name. |
| test_type: type of test results. |
| incremental: incremental json file data to merge. |
| |
| Returns: |
| Large TestFile object if update succeeds or |
| None on failure. |
| """ |
| small_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE_SMALL, JSON_RESULTS_MAX_BUILDS_SMALL) |
| large_file_updated = cls.update_file(master, builder, test_type, incremental, JSON_RESULTS_FILE, JSON_RESULTS_MAX_BUILDS) |
| |
| return small_file_updated and large_file_updated |
| |
| @classmethod |
| def update_file(cls, master, builder, test_type, incremental, filename, num_runs): |
| files = TestFile.get_files(master, builder, test_type, filename) |
| if files: |
| file = files[0] |
| new_results = cls.merge(builder, file.data, incremental, num_runs) |
| else: |
| # Use the incremental data if there is no aggregated file to merge. |
| file = TestFile() |
| file.master = master |
| file.builder = builder |
| file.test_type = test_type |
| file.name = filename |
| new_results = incremental |
| logging.info("No existing json results, incremental json is saved.") |
| |
| if not new_results or not file.save(new_results): |
| logging.info( |
| "Update failed, master: %s, builder: %s, test_type: %s, name: %s." % |
| (master, builder, test_type, filename)) |
| return False |
| |
| return True |
| |
| @classmethod |
| def get_test_list(cls, builder, json_file_data): |
| """Get list of test names from aggregated json file data. |
| |
| Args: |
| json_file_data: json file data that has all test-data and |
| non-test-data. |
| |
| Returns: |
| json file with test name list only. The json format is the same |
| as the one saved in datastore, but all non-test-data and test detail |
| results are removed. |
| """ |
| |
| logging.debug("Loading test results json...") |
| json = cls._load_json(json_file_data) |
| if not json: |
| return None |
| |
| logging.debug("Checking test results json...") |
| if not cls._check_json(builder, json): |
| return None |
| |
| test_list_json = {} |
| tests = json[builder][JSON_RESULTS_TESTS] |
| test_list_json[builder] = { |
| "tests": dict.fromkeys(tests, {})} |
| |
| return cls._generate_file_data(test_list_json) |
