#!/usr/bin/env bash # Copyright (C) 2020 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -eo pipefail API_BENCH_EXECUTABLE=$(basename "$0") pushd "$(dirname "$0")" &> /dev/null API_BENCH_DIR=$PWD popd &> /dev/null BENCHMARKS=() BUILD_DIRECTORIES=() RICHARDS_JS=-1 RICHARDS_SWIFT=-1 shouldBuild=true verbose=false iterations=5 log() { if [ "$verbose" == true ]; then echo "$*" fi } printUsage() { cat <] [ ...] EOF exit 0 } collectFlags() { while [ $# -gt 0 ]; do case "$1" in -h|--help) printUsage ;; --no-build) shouldBuild=false ;; --verbose) verbose=true ;; --iterations) shift iterations=$1 ;; *) BUILD_DIRECTORIES=("$@") break 2 ;; esac shift done if [ ${#BUILD_DIRECTORIES[@]} -lt 1 ]; then printUsage exit 1 fi for i in "${!BUILD_DIRECTORIES[@]}"; do local buildDirectory buildDirectory=${BUILD_DIRECTORIES[$i]} if ! [ -d "$buildDirectory" ]; then echo "No such file or directory: $buildDirectory" exit 1 fi pushd "$buildDirectory" &> /dev/null BUILD_DIRECTORIES[$i]=$PWD popd &> /dev/null done } collectBenchmarks() { local benchmarkDirs benchmarkDirs=("$API_BENCH_DIR"/*/) for benchmarkID in "${!benchmarkDirs[@]}"; do local benchmark benchmark=${benchmarkDirs[$benchmarkID]} pushd "$benchmark" &> /dev/null BENCHMARKS[$benchmarkID]=$PWD case "$(basename "$PWD")" in RichardsJS) RICHARDS_JS=$benchmarkID ;; RichardsSwift) RICHARDS_SWIFT=$benchmarkID ;; esac popd &> /dev/null done } forEachBenchmark() { local functor functor="$1" declare "$functor" for benchmarkID in "${!BENCHMARKS[@]}"; do local benchmark benchmark=${BENCHMARKS[$benchmarkID]} pushd "$benchmark" &> /dev/null BENCHMARK_ID=$benchmarkID BENCHMARK_NAME=$(basename "$benchmark") "$functor" popd &> /dev/null done } buildBenchmark() { project=$BENCHMARK_NAME.xcodeproj if ! [ -e "$project" ]; then return fi log "Building $BENCHMARK_NAME..." xcodebuild -project "$project" -target "$BENCHMARK_NAME" -configuration Release build } build() { forEachBenchmark "buildBenchmark" } runJSBenchmark() { "$DYLD_FRAMEWORK_PATH/jsc" "$BENCHMARK_NAME.js" } runNativeBenchmark() { pushd "build/Release" &> /dev/null "./$BENCHMARK_NAME" popd &> /dev/null } runBenchmarkIteration() { if [ -e "build" ]; then runNativeBenchmark else runJSBenchmark fi } runBenchmark() { log "Running $BENCHMARK_NAME..." local result result=$(runBenchmarkIteration) RESULTS[$((BENCHMARK_ID * iterations + ITERATION))]=$result log "Finished in ${result}ms" } computeAverage() { local benchmarkID benchmark sum avg benchmarkID=$1 benchmark=${BENCHMARKS[$benchmarkID]} sum=0 for iteration in $(seq 0 1 $((iterations - 1))); do local result result=${RESULTS[$((benchmarkID * iterations + iteration))]} sum=$((sum + result)) done bc <<< "$sum.0 / $iterations.0" } printAverage() { local benchmarkID benchmark name avg benchmarkID=$1 avg=$2 benchmark=${BENCHMARKS[$benchmarkID]} name=$(basename "$benchmark") echo "$name: ${avg}ms" } printBenchmark() { printAverage "$@" "$(computeAverage "$@")" } printResults() { local buildDirectory buildDirectory=$1 echo "Results for $buildDirectory ($iterations iterations)" echo "================================================================================" echo "" echo "References:" echo "----------------------------------------" printBenchmark "$RICHARDS_JS" printBenchmark "$RICHARDS_SWIFT" echo "" echo "Tests:" echo "----------------------------------------" local score count score=1 count=0 for benchmarkID in "${!BENCHMARKS[@]}"; do if [ "$benchmarkID" -eq "$RICHARDS_JS" ] || [ "$benchmarkID" -eq "$RICHARDS_SWIFT" ]; then continue fi avg=$(computeAverage "$benchmarkID") printAverage "$benchmarkID" "$avg" count=$((count + 1)) score=$((score * avg)) done score=$(echo "$score" "$count" | awk '{ print(5000.0 / ($1.0 ^ (1.0 / $2))) }') echo "" echo "Score: $score" } run() { local buildDirectory buildDirectory=$1 export DYLD_FRAMEWORK_PATH=$buildDirectory for iteration in $(seq 0 1 $((iterations-1))); do ITERATION=$iteration forEachBenchmark "runBenchmark" done printResults "$buildDirectory" } main() { collectFlags "$@" collectBenchmarks if [ "$shouldBuild" == true ]; then build fi for buildDirectory in "${BUILD_DIRECTORIES[@]}"; do run "$buildDirectory" done } main "$@"