# Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from webkitpy.layout_tests.layout_package import json_results_generator from webkitpy.layout_tests.models import test_expectations from webkitpy.common.iteration_compatibility import itervalues class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGenerator): """A JSON results generator for layout tests.""" LAYOUT_TESTS_PATH = "LayoutTests" # Additional JSON fields. WONTFIX = "wontfixCounts" FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGenerator.PASS_RESULT, test_expectations.SKIP: json_results_generator.JSONResultsGenerator.SKIP_RESULT, test_expectations.CRASH: "C", test_expectations.TIMEOUT: "T", test_expectations.IMAGE: "I", test_expectations.TEXT: "F", test_expectations.AUDIO: "A", test_expectations.MISSING: "O", test_expectations.LEAK: "L", test_expectations.IMAGE_PLUS_TEXT: "Z"} def __init__(self, port, results_file_base_path, expectations_by_type, run_results, test_type=""): """Modifies the results.json file. Grabs it off the archive directory if it is not found locally. Args: run_results: TestRunResults object storing the details of the test run. """ super(JSONLayoutResultsGenerator, self).__init__( port, results_file_base_path, {}, port.repository_paths(), test_type) self._expectations = expectations_by_type self._run_results = run_results self._failures = dict((test_name, run_results.results_by_name[test_name].type) for test_name in run_results.failures_by_name) self._test_timings = run_results.results_by_name def _get_path_relative_to_layout_test_root(self, test): """Returns the path of the test relative to the layout test root. For example, for: src/third_party/WebKit/LayoutTests/fast/forms/foo.html We would return fast/forms/foo.html """ index = test.find(self.LAYOUT_TESTS_PATH) if index != -1: index += len(self.LAYOUT_TESTS_PATH) if index == -1: # Already a relative path. relativePath = test else: relativePath = test[index + 1:] # Make sure all paths are unix-style. return relativePath.replace('\\', '/') # override def _get_test_timing(self, test_name): if test_name in self._test_timings: # Floor for now to get time in seconds. return int(self._test_timings[test_name].test_run_time) return 0 # override def _get_failed_test_names(self): return set(self._failures.keys()) # override def _get_modifier_char(self, test_name): if test_name not in self._run_results.results_by_name: return self.NO_DATA_RESULT if test_name in self._failures: return self.FAILURE_TO_CHAR[self._failures[test_name]] return self.PASS_RESULT # override def _get_result_char(self, test_name): return self._get_modifier_char(test_name) # override def _insert_failure_summaries(self, results_for_builder): run_results = self._run_results self._insert_item_into_raw_list(results_for_builder, len((set(run_results.failures_by_name.keys()) | run_results.tests_by_expectation[test_expectations.SKIP]) & run_results.tests_by_timeline[test_expectations.NOW]), self.FIXABLE_COUNT) self._insert_item_into_raw_list(results_for_builder, self._get_failure_summary_entry(test_expectations.NOW), self.FIXABLE) num_fixable = 0 for expectation in itervalues(self._expectations): num_fixable += len(expectation.model().get_tests_with_timeline(test_expectations.NOW)) self._insert_item_into_raw_list(results_for_builder, num_fixable, self.ALL_FIXABLE_COUNT) self._insert_item_into_raw_list(results_for_builder, self._get_failure_summary_entry(test_expectations.WONTFIX), self.WONTFIX) # override def _normalize_results_json(self, test, test_name, tests): super(JSONLayoutResultsGenerator, self)._normalize_results_json( test, test_name, tests) # Remove tests that don't exist anymore. full_path = self._filesystem.join(self._port.layout_tests_dir(), test_name) full_path = self._filesystem.normpath(full_path) if not self._filesystem.exists(full_path): del tests[test_name] def _get_failure_summary_entry(self, timeline): """Creates a summary object to insert into the JSON. Args: timeline current test_expectations timeline to build entry for (e.g., test_expectations.NOW, etc.) """ entry = {} run_results = self._run_results timeline_tests = run_results.tests_by_timeline[timeline] entry[self.SKIP_RESULT] = len( run_results.tests_by_expectation[test_expectations.SKIP] & timeline_tests) entry[self.PASS_RESULT] = len( run_results.tests_by_expectation[test_expectations.PASS] & timeline_tests) for failure_type in run_results.tests_by_expectation.keys(): if failure_type not in self.FAILURE_TO_CHAR: continue count = len(run_results.tests_by_expectation[failure_type] & timeline_tests) entry[self.FAILURE_TO_CHAR[failure_type]] = count return entry