# Copyright (C) 2017 Igalia S.L. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import logging import os from webkitpy.common.webkit_finder import WebKitFinder from webkitpy.common.test_expectations import TestExpectations from webkitpy.webdriver_tests.webdriver_driver import create_driver from webkitpy.webdriver_tests.webdriver_test_runner_selenium import WebDriverTestRunnerSelenium from webkitpy.webdriver_tests.webdriver_test_runner_w3c import WebDriverTestRunnerW3C _log = logging.getLogger(__name__) class WebDriverTestRunner(object): RUNNER_CLASSES = (WebDriverTestRunnerSelenium, WebDriverTestRunnerW3C) def __init__(self, port): self._port = port _log.info('Using port %s' % self._port.name()) _log.info('Test configuration: %s' % self._port.test_configuration()) _log.info('Using display server %s' % (self._port._display_server)) self._display_driver = self._port._driver_class()(self._port, worker_number=0, pixel_tests=False, no_timeout=True) if not self._display_driver.check_driver(self._port): raise RuntimeError("Failed to check driver %s" % self._display_driver.__class__.__name__) driver = create_driver(self._port) _log.info('Using driver at %s' % (driver.binary_path())) _log.info('Browser: %s' % (driver.browser_name())) _log.info('Parsing expectations') self._tests_dir = WebKitFinder(self._port.host.filesystem).path_from_webkit_base('WebDriverTests') expectations_file = os.path.join(self._tests_dir, 'TestExpectations.json') build_type = 'Debug' if self._port.get_option('debug') else 'Release' self._expectations = TestExpectations(self._port.name(), expectations_file, build_type) for test in self._expectations._expectations.keys(): if not os.path.isfile(os.path.join(self._tests_dir, test)): _log.warning('Test %s does not exist' % test) env = self._display_driver._setup_environ_for_test() self._runners = [runner_cls(self._port, driver, env, self._expectations) for runner_cls in self.RUNNER_CLASSES] def teardown(self): self._display_driver.stop() def run(self, tests=[]): runner_tests = [runner.collect_tests(tests) for runner in self._runners] collected_count = sum([len(tests) for tests in runner_tests]) if not collected_count: _log.info('No tests found') return 0 _log.info('Collected %d test files' % collected_count) for i in range(len(self._runners)): if runner_tests[i]: self._runners[i].run(runner_tests[i]) def process_results(self): results = {} expected_count = 0 passed_count = 0 failures_count = 0 timeout_count = 0 test_results = [] for runner in self._runners: test_results.extend(runner.results()) for result in test_results: if result.status == 'OK': for subtest, status, _, _ in result.subtest_results: if status in ['PASS', 'SKIP', 'XFAIL']: expected_count += 1 elif status in ['FAIL', 'ERROR']: results.setdefault('FAIL', []).append(os.path.join(os.path.dirname(result.test), subtest)) failures_count += 1 elif status == 'TIMEOUT': results.setdefault('TIMEOUT', []).append(os.path.join(os.path.dirname(result.test), subtest)) timeout_count += 1 elif status in ['XPASS', 'XPASS_TIMEOUT']: results.setdefault(status, []).append(os.path.join(os.path.dirname(result.test), subtest)) passed_count += 1 elif result.status == 'ERROR': # Harness execution error results.setdefault('FAIL', []).append(result.test) else: # FIXME: handle other results. pass _log.info('') retval = 0 if not results: _log.info('All tests run as expected') return retval _log.info('%d tests ran as expected, %d didn\'t\n' % (expected_count, failures_count + timeout_count + passed_count)) def report(status, actual, expected=None): retval = 0 if status not in results: return retval tests = results[status] tests_count = len(tests) if expected is None: _log.info('Unexpected %s (%d)' % (actual, tests_count)) retval += tests_count else: _log.info('Expected to %s, but %s (%d)' % (expected, actual, tests_count)) for test in tests: _log.info(' %s' % test) _log.info('') return retval report('XPASS', 'passed', 'fail') report('XPASS_TIMEOUT', 'passed', 'timeout') retval += report('FAIL', 'failures') retval += report('TIMEOUT', 'timeouts') return retval def dump_results_to_json_file(self, output_path): json_results = {} json_results['results'] = [] test_results = [] for runner in self._runners: test_results.extend(runner.results()) for result in test_results: results = {} results['test'] = result.test results['status'] = result.status results['message'] = result.message results['subtests'] = [] for name, status, message, _ in result.subtest_results: subtest = {} subtest['name'] = name.split('::', 1)[1] subtest['status'] = status subtest['message'] = message results['subtests'].append(subtest) json_results['results'].append(results) directory = os.path.dirname(output_path) if directory and not os.path.exists(directory): os.makedirs(directory) with open(output_path, 'w') as fp: json.dump(json_results, fp)