# Copyright (C) 2018 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os import logging import time from webkitpy.common import message_pool from webkitpy.port.server_process import ServerProcess, _log as server_process_logger from webkitpy.xcode.simulated_device import SimulatedDeviceManager class Runner(object): STATUS_PASSED = 0 STATUS_FAILED = 1 STATUS_CRASHED = 2 STATUS_TIMEOUT = 3 STATUS_DISABLED = 4 STATUS_RUNNING = 5 NAME_FOR_STATUS = [ 'Passed', 'Failed', 'Crashed', 'Timeout', 'Disabled', ] def __init__(self, port, printer): self.port = port self.printer = printer self.tests_run = 0 self._num_workers = 1 self._has_logged_for_test = True # Suppress an empty line between "Running tests" and the first test's output. self.results = {} @staticmethod def _shard_tests(tests): shards = {} for test in tests: shard_prefix = '.'.join(test.split('.')[:-1]) if shard_prefix not in shards: shards[shard_prefix] = [] shards[shard_prefix].append(test) return shards # FIXME API tests should run as an app, we won't need this function @staticmethod def command_for_port(port, args): if (port.get_option('force')): args.append('--gtest_also_run_disabled_tests=1') if 'simulator' in port.port_name: assert SimulatedDeviceManager.INITIALIZED_DEVICES return ['/usr/bin/xcrun', 'simctl', 'spawn', SimulatedDeviceManager.INITIALIZED_DEVICES[0].udid] + args elif 'device' in port.port_name: raise RuntimeError('Running api tests on {} is not supported'.format(port.port_name)) elif port.host.platform.is_win(): args[0] = os.path.splitext(args[0])[0] + '.exe' return args def run(self, tests, num_workers): if not tests: return self.printer.write_update('Sharding tests ...') shards = Runner._shard_tests(tests) original_level = server_process_logger.level server_process_logger.setLevel(logging.CRITICAL) try: self._num_workers = min(num_workers, len(shards)) with message_pool.get(self, lambda caller: _Worker(caller, self.port, shards), self._num_workers) as pool: pool.run(('test', shard) for shard, _ in shards.iteritems()) finally: server_process_logger.setLevel(original_level) def handle(self, message_name, source, test_name=None, status=0, output=''): if message_name == 'did_spawn_worker': return source = '' if self._num_workers == 1 else source + ' ' will_stream_logs = self._num_workers == 1 and self.port.get_option('verbose') if message_name == 'ended_test': update = '{}{} {}'.format(source, test_name, Runner.NAME_FOR_STATUS[status]) # Don't print test output if --quiet. if status != Runner.STATUS_PASSED or (output and not self.port.get_option('quiet')): if not will_stream_logs: for line in output.splitlines(): if not self._has_logged_for_test: self._has_logged_for_test = True self.printer.writeln(source) self.printer.writeln('{} {}'.format(source, line)) self.printer.writeln(update) else: self.printer.write_update(update) self.tests_run += 1 self.results[test_name] = (status, output) self._has_logged_for_test = False if message_name == 'log' and will_stream_logs: for line in output.splitlines(): if not self._has_logged_for_test: self._has_logged_for_test = True self.printer.writeln(source) self.printer.writeln('{} {}'.format(source, line)) def result_map_by_status(self, status=None): map = {} for test_name, result in self.results.iteritems(): if result[0] == status: map[test_name] = result[1] return map class _Worker(object): def __init__(self, caller, port, shard_map): self._caller = caller self._port = port self.host = port.host self._shard_map = shard_map # ServerProcess doesn't allow for a timeout of 'None,' this uses a week instead of None. self._timeout = int(self._port.get_option('timeout')) if self._port.get_option('timeout') else 60 * 24 * 7 @staticmethod def _filter_noisy_output(output): result = '' for line in output.splitlines(): if line.lstrip().startswith('objc['): continue result += line + '\n' return result def _run_single_test(self, binary_name, test): server_process = ServerProcess( self._port, binary_name, Runner.command_for_port(self._port, [self._port._build_path(binary_name), '--gtest_filter={}'.format(test)]), env=self._port.environment_for_api_tests()) status = Runner.STATUS_RUNNING if test.split('.')[1].startswith('DISABLED_') and not self._port.get_option('force'): status = Runner.STATUS_DISABLED try: deadline = time.time() + self._timeout if status != Runner.STATUS_DISABLED: server_process.start() stdout_buffer = '' stderr_buffer = '' while status == Runner.STATUS_RUNNING: stdout_line, stderr_line = server_process.read_either_stdout_or_stderr_line(deadline) if not stderr_line and not stdout_line: break if stderr_line: stderr_buffer += stderr_line self.post('log', output=stderr_line[:-1]) if stdout_line: if '**PASS**' in stdout_line: status = Runner.STATUS_PASSED elif '**FAIL**' in stdout_line: status = Runner.STATUS_FAILED else: stdout_buffer += stdout_line self.post('log', output=stdout_line[:-1]) if status == Runner.STATUS_DISABLED: pass elif server_process.timed_out: status = Runner.STATUS_TIMEOUT elif server_process.has_crashed(): status = Runner.STATUS_CRASHED elif status == Runner.STATUS_RUNNING: status = Runner.STATUS_FAILED finally: remaining_stderr = server_process.pop_all_buffered_stderr() remaining_stdout = server_process.pop_all_buffered_stdout() self.post('log', output=remaining_stderr + remaining_stdout) output_buffer = stderr_buffer + stdout_buffer + remaining_stderr + remaining_stdout server_process.stop() self.post('ended_test', '{}.{}'.format(binary_name, test), status, self._filter_noisy_output(output_buffer)) def _run_shard_with_binary(self, binary_name, tests): remaining_tests = list(tests) # Try to run the shard in a single process. while remaining_tests and not self._port.get_option('run_singly'): starting_length = len(remaining_tests) server_process = ServerProcess( self._port, binary_name, Runner.command_for_port(self._port, [self._port._build_path(binary_name), '--gtest_filter={}'.format(':'.join(remaining_tests))]), env=self._port.environment_for_api_tests()) try: deadline = time.time() + self._timeout last_test = None last_status = None stdout_buffer = '' server_process.start() while remaining_tests: stdout = server_process.read_stdout_line(deadline) # If we've triggered a timeout, we don't know which test caused it. Break out and run singly. if stdout is None and server_process.timed_out: break if stdout is None and server_process.has_crashed(): # It's possible we crashed before printing anything. if last_status == Runner.STATUS_PASSED: last_test = None else: last_status = Runner.STATUS_CRASHED break assert stdout is not None stdout_split = stdout.rstrip().split(' ') if len(stdout_split) != 2 or not (stdout_split[0].startswith('**') and stdout_split[0].endswith('**')): stdout_buffer += stdout continue if last_test is not None: remaining_tests.remove(last_test) self.post('ended_test', '{}.{}'.format(binary_name, last_test), last_status, stdout_buffer) deadline = time.time() + self._timeout stdout_buffer = '' if '**PASS**' == stdout_split[0]: last_status = Runner.STATUS_PASSED else: last_status = Runner.STATUS_FAILED last_test = stdout_split[1] # We assume that stderr is only relevant if there is a crash (meaning we triggered an assert) if last_test: remaining_tests.remove(last_test) stdout_buffer += server_process.pop_all_buffered_stdout() stderr_buffer = server_process.pop_all_buffered_stderr() if last_status == Runner.STATUS_CRASHED else '' self.post('log', output=stdout_buffer + stderr_buffer) self.post('ended_test', '{}.{}'.format(binary_name, last_test), last_status, self._filter_noisy_output(stdout_buffer + stderr_buffer)) if server_process.timed_out: break # If we weren't able to determine the results for any tests, we need to run what remains singly. if starting_length == len(remaining_tests): break finally: server_process.stop() # Now, just try and run the rest of the tests singly. for test in remaining_tests: self._run_single_test(binary_name, test) def post(self, message_name, test_name=None, status=0, output=''): self._caller.post(message_name, test_name, status, output) def handle(self, message_name, source, shard_name): assert message_name == 'test' self.post('started_shard', shard_name) binary_map = {} for test in self._shard_map[shard_name]: split_test_name = test.split('.') if split_test_name[0] not in binary_map: binary_map[split_test_name[0]] = [] binary_map[split_test_name[0]].append('.'.join(split_test_name[1:])) for binary_name, test_list in binary_map.iteritems(): self._run_shard_with_binary(binary_name, test_list)