Refactor wasm-engines benchmarks to run outside container.

Makefile

@@ -26,9 +26,7 @@ benchmark_scout_engines:
 
 benchmark_wasm_engines:
 	cd wasm-engines && \
-	./run_benchmarks.sh && \
-	cp benchmark_results_data/standalone_wasm_results.csv ../benchmark_results_data/ && \
-	cp benchmark_results/data/native_results.csv ../benchmark_results_data/
+	./run.sh
 
 benchmark: benchmark_scout_engines benchmark_wasm_engines benchmark_evm_engines benchmark_evm_precompiles
 

charts/generate-charts.py

@@ -1540,7 +1540,7 @@ def splitName(test_name):
 native_results = {}
 for i in range(len(df_native_raw)):
     test_name = df_native_raw['test_name'][i]
-    native_results[test_name] = df_native_raw['elapsed_times'][i].split(', ')
+    native_results[test_name] = df_native_raw['elapsed_times'][i].strip(' ').split(',')
 
 
 test_names = df_native_raw['test_name'].tolist()

wasm-engines/bench_wasm_and_native.sh → wasm-engines/prep_rust.sh

@@ -1,13 +1,7 @@
-#!/usr/bin/env bash
-
-# first compile standalone wasm files from rust code, and benchmark native rust
-# later, benchmark the standalone wasm files in all the wasm engines
-
-set -e
+cd /benchprep
 
-# result output paths should be in mounted docker volumes
-CSV_NATIVE_RESULTS=/benchmark_results_data/native_benchmarks.csv
-CSV_WASM_RESULTS=/benchmark_results_data/standalone_wasm_results.csv
+# CSV_NATIVE_RESULTS=$OUTPUT_DIR/native_benchmarks.csv
+# CSV_WASM_RESULTS=$OUTPUT_DIR/standalone_wasm_results.csv
 
 # benchnativerust_prepwasm.py will use rust code templates and input vectors to
 # prepare standalone wasm files and native rust executables
@@ -19,17 +13,12 @@ WASM_FILE_DIR=/wasmfiles
 
 # files in WASM_FILE_DIR will be minified, and outputted to WASM_MINIFIED_DIR
 # these files will be benchmarked in all the engines
-WASM_MINIFIED_DIR=/wasmfilesminified
 
+WASM_MINIFIED_DIR=/wasmfilesminified
 
-# save cpu info to a file, so we know what machine was used to run the benchmarks
-grep -E '^model name|^cpu MHz' /proc/cpuinfo > /benchmark_results_data/cpuinfo.txt
-
+# Fill rust code and build it
 
-# fill rust code templates with input vectors. compile rust code to wasm and native. benchmark native
-# wasm will be further processed and benchmarked later
-cd /benchprep
-python3.8 benchnativerust_prepwasm.py --wasmoutdir="${WASM_FILE_DIR}" --csvresults="${CSV_NATIVE_RESULTS}" --rustcodedir="${RUST_CODE_DIR}" --inputvectorsdir="${INPUT_VECTORS_DIR}"
+python3.8 fill_rust.py --wasmoutdir="${WASM_FILE_DIR}" --rustcodedir="${RUST_CODE_DIR}" --inputvectorsdir="${INPUT_VECTORS_DIR}"
 
 # furthur process wasm files by minifying them.
 
@@ -53,8 +42,3 @@ do
   dest="${WASM_MINIFIED_DIR}/${filename}"
   /root/sentinel-minify-tool/wasm-utils/target/debug/wasm-minify "${filename}" "$dest"
 done
-
-echo "running benchmarks"
-cd /benchrunner
-python3.8 main.py --wasmdir="${WASM_MINIFIED_DIR}" --csvfile="${CSV_WASM_RESULTS}"
-chown -R 1000:1000 /benchrunner /benchprep /benchmark_results_data

wasm-engines/results/cpuinfo.txt

@@ -0,0 +1,8 @@
+model name	: Intel(R) Core(TM) i5-6600K CPU @ 3.50GHz
+cpu MHz		: 2013.215
+model name	: Intel(R) Core(TM) i5-6600K CPU @ 3.50GHz
+cpu MHz		: 2936.006
+model name	: Intel(R) Core(TM) i5-6600K CPU @ 3.50GHz
+cpu MHz		: 2583.646
+model name	: Intel(R) Core(TM) i5-6600K CPU @ 3.50GHz
+cpu MHz		: 2611.170

wasm-engines/run.sh

@@ -0,0 +1,75 @@
+#! /bin/env bash
+
+# first compile standalone wasm files from rust code, and benchmark native rust
+# later, benchmark the standalone wasm files in all the wasm engines
+
+set -e
+
+# result output paths should be in mounted docker volumes
+CSV_NATIVE_RESULTS=$BENCHMARK_RESULTS_DIR/native_benchmarks.csv
+CSV_WASM_RESULTS=$BENCHMARK_RESULTS_DIR/standalone_wasm_results.csv
+
+RESULTS_DIR=$(pwd)/../benchmark_results_data/wasm-results #./benchmark_results_data
+RUST_CODE_DIR=$(pwd)/rust-code
+WASM_SOURCE_DIR=$(pwd)/wasmfiles # we don't have corresponding source code for some of the wasm benchmarking blobs
+INPUT_VECTORS_DIR=$(pwd)/inputvectors
+WASM_FILE_DIR=$RESULTS_DIR/wasmfiles
+WASM_MINIFIED_DIR=$RESULTS_DIR/wasmfilesminified
+RUST_BIN_DIR=$RESULTS_DIR/rust_bin
+WASM_ENGINE_BIN_DIR=$RESULTS_DIR/wasm_engine_bin
+
+rm -rf $RESULTS_DIR && mkdir $RESULTS_DIR
+mkdir $WASM_FILE_DIR
+mkdir $WASM_MINIFIED_DIR
+mkdir $RUST_BIN_DIR
+mkdir $WASM_ENGINE_BIN_DIR
+
+cp -r $WASM_SOURCE_DIR/* $WASM_FILE_DIR
+
+grep -E '^model name|^cpu MHz' /proc/cpuinfo > $RESULTS_DIR/cpuinfo.txt
+
+echo "TODO some standalone wasm benchmarks are missing their source, incorporate them.  (don't delete the entire results directory on each run"
+
+# copy all wasm engine binaries to the host machine from the container images where they were built
+docker cp $(docker run -d -t ewasm/asmble:1 sleep 3s):/asmble/bin/asmble $WASM_ENGINE_BIN_DIR/asmble
+docker cp $(docker run -d -t ewasm/fizzy:1 sleep 3s):/fizzy/build/bin/fizzy-bench $WASM_ENGINE_BIN_DIR/fizzy-bench
+docker cp $(docker run -d -t ewasm/life:1 sleep 3s):/life/life $WASM_ENGINE_BIN_DIR/life
+docker cp $(docker run -d -t ewasm/ssvm:1 sleep 3s):/SSVM/build/tools/ssvm/ssvm $WASM_ENGINE_BIN_DIR/ssvm
+docker cp $(docker run -d -t ewasm/vanilla-wabt:1 sleep 3s):/vanilla-wabt/build/wasm-interp $WASM_ENGINE_BIN_DIR/vanilla-wabt
+docker cp $(docker run -d -t ewasm/wabt:1 sleep 3s):/wabt/build/wasm-interp $WASM_ENGINE_BIN_DIR/wabt
+docker cp $(docker run -d -t ewasm/wagon:1 sleep 3s):/wagon/cmd/wasm-run/wasm-run $WASM_ENGINE_BIN_DIR/wagon
+
+docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/product-mini/platforms/linux/build_interp/iwasm $WASM_ENGINE_BIN_DIR/iwasm
+docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/wamr-compiler/build/wamrc $WASM_ENGINE_BIN_DIR/wamrc
+
+docker cp $(docker run -d -t ewasm/wasm3:1 sleep 3s):/wasm3/build/wasm3 $WASM_ENGINE_BIN_DIR/wasm3
+
+docker cp $(docker run -d -t ewasm/wasmi:1 sleep 3s):/wasmi/target/release/invoke $WASM_ENGINE_BIN_DIR/invoke
+
+docker cp $(docker run -d -t ewasm/wasmtime:1 sleep 3s):/wasmtime/target/release/wasmtime $WASM_ENGINE_BIN_DIR/wasmtime
+
+docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-compile $WASM_ENGINE_BIN_DIR/wavm-compile
+docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-run $WASM_ENGINE_BIN_DIR/wavm-run
+
+docker run -v $RUST_BIN_DIR:/rust-bin -v $INPUT_VECTORS_DIR:/inputvectors -v $RESULTS_DIR:/results -v $(pwd)/rust-code:/rust-code -v $(pwd)/scripts:/scripts -it ewasm/bench-build-base:1 bash /scripts/fill_rust.sh
+
+ln -s $(which node) $WASM_ENGINE_BIN_DIR/node
+
+# copy scripts which invoke engines to wasm_bin_dir (fizzy and wamr-aot)
+cp scripts/fizzy.sh $WASM_ENGINE_BIN_DIR/
+cp scripts/wamr_aot.sh $WASM_ENGINE_BIN_DIR/
+
+sudo chown -R 1000:1000 $RESULTS_DIR
+
+echo "TODO minify rust wasm"
+
+# run rust (native) benchmarks
+
+python3 scripts/bench_native.py --rustbindir=$RUST_BIN_DIR --csvresults=$RESULTS_DIR
+
+# run wasm benchmarks
+
+python3 scripts/bench_wasm.py --wasmenginedir $WASM_ENGINE_BIN_DIR --csvfile $RESULTS_DIR/wasm_engines.csv --wasmdir $WASM_FILE_DIR
+
+mv $RESULTS_DIR/native_benchmarks.csv $RESULTS_DIR/../native_benchmarks.csv
+mv $RESULTS_DIR/wasm_engines.csv $RESULTS_DIR/../standalone_wasm_results.csv

wasm-engines/scripts/Pipfile

@@ -0,0 +1,18 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+matplotlib = "==3.2.1"
+pandas = "==1.0.3"
+numpy = "==1.18.2"
+durationpy = "==0.5"
+jupyter = "==1.0.0"
+seaborn = "==0.10.1"
+adjustText = "==0.7.3"
+
+[requires]
+python_version = "3.8"

wasm-engines/scripts/bench_native.py

@@ -0,0 +1,68 @@
+import jinja2, json, re
+from functools import reduce
+import subprocess
+import nanodurationpy as durationpy
+import csv
+import time
+import datetime
+import os
+import shutil
+import glob
+import argparse
+import sys
+
+# how many times to run native exec
+RUST_BENCH_REPEATS = 50
+
+def bench_rust_binary(native_exec):
+    bench_times = []
+    for i in range(1,RUST_BENCH_REPEATS):
+        rust_process = subprocess.Popen(native_exec, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True)
+        rust_process.wait(None)
+        stdoutlines = [str(line, 'utf8') for line in rust_process.stdout]
+        print(("").join(stdoutlines), end="")
+        elapsedline = stdoutlines[0]
+        elapsedmatch = re.search("Time elapsed in bench\(\) is: ([\w\.]+)", elapsedline)
+        elapsed_time = durationpy.from_str(elapsedmatch[1])
+        bench_times.append(elapsed_time.total_seconds())
+    return bench_times
+
+def write_output_to_csv(native_benchmarks, result_file):
+    # TODO backup old results
+
+    print("TODO: reincorporate native_file_size column for native_benchmarks csv output if it is actually used anywhere")
+
+    with open(result_file, 'w', newline='') as bench_result_file:
+        fieldnames = ['test_name', 'elapsed_times', 'native_file_size']
+        writer = csv.DictWriter(bench_result_file, fieldnames=fieldnames)
+        writer.writeheader()
+        for item in native_benchmarks:
+            times_str = item['elapsed_times']
+            test_name = item['test_name']
+            writer.writerow({"test_name" : test_name, "elapsed_times" : times_str})
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--rustbindir', help='directory containing executables for standalone benchmarks compiled from rust')
+    parser.add_argument('--csvresults', help='full path of csv result file')
+    args = vars(parser.parse_args())
+
+    rust_native_dir = os.path.join(os.getcwd(), 'results/bin')
+    # logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s %(message)s',
+    #                 datefmt='%m/%d/%Y %I:%M:%S %p')
+
+    # logger = logging.getLogger("wasm_bench_logger")
+
+    executables = glob.glob(args['rustbindir'] + "/*")
+
+    results = []
+
+    for binary in executables:
+        result_times = bench_rust_binary(binary)
+        result_times = ",".join(list(map(lambda x: str(x), result_times)))
+        results.append({"test_name": os.path.basename(binary).strip("_native"), "elapsed_times": result_times})
+
+    write_output_to_csv(results, os.path.join(args['csvresults'], "native_benchmarks.csv"))
+
+if __name__ == "__main__":
+    main()

wasm-engines/main.py → wasm-engines/scripts/bench_wasm.py

@@ -24,6 +24,7 @@
 parser.add_argument('--wasmdir', help='full path of dir containing wasm files')
 parser.add_argument('--csvfile', help='name of csv result file')
 parser.add_argument('--engines', help='comma-separated list of engines to benchmark')
+parser.add_argument('--wasmenginedir', help='directory containing executables for all wasm engines')
 
 args = vars(parser.parse_args())
 
@@ -77,6 +78,7 @@ def main():
     wasm_dir = args['wasmdir']
     csv_file_path = args['csvfile']
     engines_to_run = args['engines']
+    engine_bin_dir = args['wasmenginedir']
     # "wagon,wabt,v8-liftoff,v8-turbofan,v8-interpreter,wasmtime,wavm,life-polymerase,life,wasmi,asmble"
     vms_to_run = {}
     if engines_to_run is not None:
@@ -87,13 +89,14 @@ def main():
         vms_to_run = vm_descriptors
 
     print("vms_to_run:", vms_to_run)
-    ## TODO: print version of each engine
 
     vm_bencher = WasmVMBencher()
     test_descriptors = getTestDescriptors(wasm_dir)
-    test_results = vm_bencher.run_tests(test_descriptors, vms_to_run)
+    # TODO no hardcode script path here
+    test_results = vm_bencher.run_tests(test_descriptors, engine_bin_dir, vms_to_run)
     print("test_results:")
     print(test_results)
+
     save_test_results(csv_file_path, test_results)