diff --git a/wasm-engines/wasm-engines.ipynb b/wasm-engines/wasm-engines.ipynb new file mode 100644 index 0000000..f55dc19 --- /dev/null +++ b/wasm-engines/wasm-engines.ipynb @@ -0,0 +1,6552 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#%matplotlib widget\n", + "\n", + "#%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.ticker as ticker\n", + "#import matplotlib\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import durationpy\n", + "import math\n", + "from os.path import join\n", + "from collections import defaultdict\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.style.use('ggplot')\n", + "\n", + "COLORS_DEFAULT ={'blue': '#348ABD', 'red': '#E24A33'}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pd.set_option('display.max_rows', 1000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# conda install -c phlya adjusttext \n", + "from adjustText import adjust_text" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## csv result files directory name and file names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "CSV_RESULT_DIR = \"../benchmark_results_data\"\n", + "\n", + "STANDALONE_WASM_RESULT_FILE = \"standalone_wasm_results.csv\"\n", + "NATIVE_RESULT_FILE = \"native_benchmarks.csv\"\n", + "SCOUT_RESULT_FILE = \"scout_bignum_benchmarks.csv\"\n", + "GETH_PRECOMPILE_RESULT_FILE = \"geth_precompile_benchmarks.csv\"\n", + "EVM_RESULT_FILE = \"evm_benchmarks.csv\"\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## To add a new engine, add the engine name used in the csv file to this list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "INTERPRETER_ENGINES = ['life', 'ssvm', 'wagon', 'wasmi', 'wabt', 'v8-interpreter', 'wasm3', 'wamr-interp', 'fizzy']\n", + "COMPILER_ENGINES = ['lifePolymerase', 'wasmtime', 'wavm', 'v8-liftoff', 'v8-turbofan', 'asmble', 'wamr-jit', 'wamr-aot']\n", + "\n", + "wasm_vm_names = INTERPRETER_ENGINES + COMPILER_ENGINES" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Helper funcs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def toMs(secs):\n", + " if secs < 0.001:\n", + " μs = secs * 1000000\n", + " return \"{}μs\".format(round(μs, 1))\n", + " if secs < 0.1:\n", + " ms = secs * 1000\n", + " return \"{}ms\".format(round(ms, 1))\n", + " elif secs < 1:\n", + " ms = secs * 1000\n", + " return \"{}ms\".format(int(round(ms, 0)))\n", + " else:\n", + " return \"{}s\".format(round(secs, 2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def labelBarHeights(ax, to_ms=True, small=False, to_factor=False, lower_y_bound=True):\n", + " labels = []\n", + " for p in ax.patches:\n", + " if p.get_height() == 0:\n", + " continue\n", + " label_val = p.get_height()\n", + " #print(\"height:\", p.get_height())\n", + " y_pos = p.get_y() + label_val\n", + " #y_offset = -10\n", + " y_offset = 0\n", + " y_bound = ax.get_ybound()[1]\n", + " if y_pos > y_bound:\n", + " y_pos = y_bound\n", + " #if label_val < (0.03 * y_bound):\n", + " # if bar is too small to fit text, place above the bar\n", + " #y_offset = 5\n", + " if lower_y_bound:\n", + " if label_val < (0.03 * y_bound) and small is False:\n", + " # don't place labels near bottom axis, adjust_text cant deal\n", + " y_offset = 0.03 * y_bound\n", + " \n", + " if label_val > 1000:\n", + " label_val = int(label_val)\n", + " if to_ms:\n", + " label_val = toMs(label_val)\n", + " if to_factor:\n", + " if not isinstance(to_factor, int):\n", + " to_factor = 2\n", + " label_val = str(round(label_val, to_factor)) + \"x\"\n", + " \"\"\"\n", + " txt_label = ax.annotate(toMs(label_val),\n", + " (p.get_x()+p.get_width()/2.,\n", + " y_pos),\n", + " ha='center',\n", + " va='center',\n", + " xytext=(0, y_offset),\n", + " textcoords='offset points',\n", + " fontsize=10,\n", + " weight=\"bold\")\n", + " \"\"\"\n", + " \n", + " txt_label = ax.text(\n", + " x=p.get_x()+p.get_width()/2.,\n", + " y=y_pos+y_offset,\n", + " s=label_val,\n", + " ha='center',\n", + " va='center',\n", + " #xytext=(0, y_offset),\n", + " #textcoords='offset points',\n", + " fontsize=10,\n", + " weight=\"bold\")\n", + "\n", + " labels.append(txt_label)\n", + "\n", + " #adjust_text(labels, ax=ax)\n", + " return labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def labelBarWidths(ax, to_ms=True, to_factor=False, to_kb=False, round_places=None):\n", + " for p in ax.patches:\n", + " label_val = p.get_width()\n", + " if label_val == 0:\n", + " continue\n", + " #print(\"bar width:\", label_val)\n", + " x_pos = p.get_x() + label_val\n", + " x_offset = -30\n", + " x_bound = ax.get_xbound()[1]\n", + " if (x_pos > x_bound):\n", + " x_pos = x_bound\n", + " #if label_val < (0.25 * x_bound):\n", + " if label_val < (0.15 * x_bound):\n", + " # if bar is too small to fit text, place next to bar\n", + " x_offset = 30\n", + " if label_val > 1000:\n", + " label_val = int(label_val)\n", + " if to_ms:\n", + " label_val = toMs(label_val)\n", + " if to_factor:\n", + " label_val = str(round(label_val, 2)) + \"x\"\n", + " if to_kb:\n", + " kbs = label_val / 1000.0\n", + " label_val = str(round(kbs, 1)) + \" kB\"\n", + " if round_places:\n", + " label_val = str(round(label_val, round_places))\n", + " ax.annotate(label_val,\n", + " (x_pos,\n", + " p.get_y()+p.get_height()/2.),\n", + " ha='center',\n", + " va='center',\n", + " xytext=(x_offset, 0),\n", + " textcoords='offset points',\n", + " fontsize=10,\n", + " weight=\"bold\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def adjust_text_labels(labels, ax=None):\n", + " adjust_text(labels, ax=ax, autoalign='y', only_move={'points':'y', 'text':'y'})\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def read_results(file_name):\n", + " results = defaultdict(list)\n", + " csv_results = pd.read_csv(file_name) \n", + " return csv_results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def filterDfEngines(df_data, engines):\n", + " df_data_filtered = df_data.copy()\n", + " for fil in engines:\n", + " df_data_filtered = df_data_filtered[df_data_filtered['engine'] != fil]\n", + "\n", + " return df_data_filtered" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## print benchmark machine cpu type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(join(CSV_RESULT_DIR, \"cpuinfo.txt\"), 'r') as cpuinfofile:\n", + " [print(line.rstrip()) for line in cpuinfofile.readlines()]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import EVM engine results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw = read_results(join(CSV_RESULT_DIR, EVM_RESULT_FILE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import scout engine results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_data = read_results(join(CSV_RESULT_DIR, SCOUT_RESULT_FILE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## functions for plotting a two engine comparison" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## we aren't using this chart style anymore. replaced by plotThreeTestsGrouped()\n", + "def plotTwoEngines(df_benches, two_engines, title=\"Title\"):\n", + " df_1 = df_benches[df_benches['engine'] == two_engines[0]]\n", + " df_2 = df_benches[df_benches['engine'] == two_engines[1]]\n", + " plt.figure()\n", + " f, (ax1) = plt.subplots(1, 2, sharey=True, figsize=(13,6))\n", + "\n", + " df_1.set_index('bench_name').sort_values('exec_time')[['exec_time']].plot.bar(ax=ax1[0], color=COLORS_DEFAULT['blue'])\n", + " ax1[0].set_title(two_engines[0])\n", + " ax1[0].set(ylabel='seconds', xlabel='')\n", + " df_2.set_index('bench_name').sort_values('exec_time')[['exec_time']].plot.bar(ax=ax1[1], color=COLORS_DEFAULT['blue'])\n", + " ax1[1].set_title(two_engines[1])\n", + " ax1[1].set(xlabel='')\n", + " adjust_text_labels(labelBarHeights(ax1[0]), ax=ax1[0])\n", + " adjust_text_labels(labelBarHeights(ax1[1]), ax=ax1[1])\n", + " f.suptitle(title, fontsize=16, y=0.98)\n", + " return plt\n", + "\n", + "\n", + "#df_sha256_1024 = df_scout_data[df_scout_data['bench_name'].str.contains('sha256_1024')]\n", + "#df_blake2b_64 = df_scout_data[df_scout_data['bench_name'].str.contains('blake2b_64')]\n", + "#plotTwoEngines(df_sha256_1024, [\"wabt-baseline\", \"wabt-with-superops\"], \"sha256 1024 bytes\")\n", + "#plotTwoEngines(df_blake2b_64, [\"wabt-baseline\", \"wabt-with-superops\"], \"blake2b 64 bytes\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotThreeTestsGrouped(df_benches, three_tests, title=\"Title\"):\n", + " df_1 = df_benches[df_benches['bench_name'].str.contains(three_tests[0])]\n", + " df_2 = df_benches[df_benches['bench_name'].str.contains(three_tests[1])]\n", + " df_3 = df_benches[df_benches['bench_name'].str.contains(three_tests[2])]\n", + " # to group by engine, pivot to `columns='enginge'`\n", + " df_1 = pd.pivot_table(df_1, values='exec_time', columns=['engine'], index=['bench_name'], aggfunc=np.mean)\n", + " df_2 = pd.pivot_table(df_2, values='exec_time', columns=['engine'], index=['bench_name'], aggfunc=np.mean)\n", + " df_3 = pd.pivot_table(df_3, values='exec_time', columns=['engine'], index=['bench_name'], aggfunc=np.mean)\n", + "\n", + " plt.figure()\n", + " f, (ax1) = plt.subplots(1, 3, sharey=True, figsize=(13,6))\n", + "\n", + " df_1.sort_values('wabt-with-superops').plot.bar(ax=ax1[0])\n", + " ax1[0].set_title(three_tests[0])\n", + " ax1[0].set(ylabel='seconds', xlabel='')\n", + "\n", + " df_2.sort_values('wabt-with-superops').plot.bar(ax=ax1[1])\n", + " ax1[1].set_title(three_tests[1])\n", + " ax1[1].set(xlabel='')\n", + "\n", + " df_3.sort_values('wabt-with-superops').plot.bar(ax=ax1[2])\n", + " ax1[2].set_title(three_tests[2])\n", + " ax1[2].set(xlabel='')\n", + "\n", + " adjust_text_labels(labelBarHeights(ax1[0]), ax=ax1[0])\n", + " adjust_text_labels(labelBarHeights(ax1[1]), ax=ax1[1])\n", + " adjust_text_labels(labelBarHeights(ax1[2]), ax=ax1[2])\n", + "\n", + " f.suptitle(title, fontsize=16, y=0.98)\n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare wabt-optimized against wabt-baseline on hash function benchmarks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TODO: temporarily disabled until new C_ewasm data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#fig_plt = plotThreeTestsGrouped(df_scout_data, [\"blake2b_64\", \"blake2b_256\", \"blake2b_1024\"], \"blake2b C implementations compared\")\n", + "\n", + "#fig_plt.savefig('../images/hashfuncs-blake2b-C-compared-wabt-baseline-vs-superops.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#fig_plt = plotThreeTestsGrouped(df_scout_data, [\"sha256_64\", \"sha256_256\", \"sha256_1024\"], \"sha256 C implementations compared\")\n", + "\n", + "#fig_plt.savefig('../images/hashfuncs-sha256-C-compared-wabt-baseline-vs-superops.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#fig_plt = plotThreeTestsGrouped(df_scout_data, [\"keccak256_64\", \"keccak256_256\", \"keccak256_1024\"], \"keccak256 C implementations compared\")\n", + "\n", + "#fig_plt.savefig('../images/hashfuncs-keccak256-C-compared-wabt-baseline-vs-superops.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## functions for plotting biturbo and bignum benchmarks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# non-stacked bar chart, for only plotting exec time or total time\n", + "def plotOneTestUsingTimeCol(df_data, time_col_name=\"time\", suptitle=\"title\", suptitle_pos=1.00, subtitle=\"subchart\", subtitle_size='medium'):\n", + " plt.style.use('ggplot')\n", + " plt.figure()\n", + " df_total_times = df_data.copy()\n", + " ax = df_total_times.sort_values(time_col_name)[[time_col_name]].plot.bar(figsize=(14,8), color=COLORS_DEFAULT['blue'])\n", + " # fontsize = {size in points, 'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n", + " ax.set_title(subtitle, fontsize=subtitle_size)\n", + " ax.set(ylabel=\"seconds\", xlabel=\"\")\n", + " ax.xaxis.set_tick_params(labelrotation=60, labelsize=12)\n", + " adjust_text_labels(labelBarHeights(ax))\n", + " plt.suptitle(suptitle, fontsize=18, y=suptitle_pos)\n", + " return plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# non-stacked bar chart, for only plotting exec time or total time\n", + "def plotOneTest(df_data, suptitle=\"title\", suptitle_pos=1.00, subtitle=\"subchart\", subtitle_size='medium'):\n", + " plt.style.use('ggplot')\n", + " plt.figure()\n", + " df_total_times = df_data[['exec_time', 'parse_time']].copy()\n", + " df_total_times.fillna(0, inplace=True)\n", + " df_total_times['time'] = df_total_times['exec_time'] + df_total_times['parse_time']\n", + " ax = df_total_times.sort_values('time')[['time']].plot.bar(figsize=(14,8), color=COLORS_DEFAULT['blue'])\n", + " # fontsize = {size in points, 'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n", + " ax.set_title(subtitle, fontsize=subtitle_size)\n", + " ax.set(ylabel=\"seconds\", xlabel=\"\")\n", + " ax.xaxis.set_tick_params(labelrotation=60, labelsize=12)\n", + " adjust_text_labels(labelBarHeights(ax))\n", + " plt.suptitle(suptitle, fontsize=18, y=suptitle_pos)\n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# non-stacked bar chart, for only plotting exec time or total time\n", + "def plotOneTestColoredTicksBLS(df_data, suptitle=\"title\", suptitle_pos=1.00, subtitle=\"subchart\", subtitle_size='medium', highlight_ticks=None):\n", + " #f = plt.figure(figsize=[14, 8])\n", + " #plt.figure(figsize=[14, 8])\n", + " plt.style.use('ggplot')\n", + " plt.figure()\n", + " df_total_times = df_data[['exec_time', 'parse_time']].copy()\n", + " df_total_times.fillna(0, inplace=True)\n", + " df_total_times['time'] = df_total_times['exec_time'] + df_total_times['parse_time']\n", + " ax = df_total_times.sort_values('time')[['time']].plot.bar(figsize=(14,8), color=COLORS_DEFAULT['blue'])\n", + " # fontsize = {size in points, 'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n", + " ax.set_title(subtitle, fontsize=subtitle_size)\n", + " ax.set(ylabel=\"seconds\", xlabel=\"\")\n", + " ax.xaxis.set_tick_params(labelrotation=85, labelsize=12)\n", + " if highlight_ticks is not None:\n", + " for highlight_tick in highlight_ticks:\n", + " for i in plt.gca().get_xticklabels():\n", + " if i.get_text() == highlight_tick:\n", + " i.set_color('tab:green')\n", + " i.set_fontstyle('oblique')\n", + " i.set_fontweight('heavy')\n", + "\n", + " # {'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan'}\n", + " #[i.set_color('tab:green') for i in plt.gca().get_xticklabels() if i.get_text() == highlight_tick]\n", + " adjust_text_labels(labelBarHeights(ax))\n", + " plt.suptitle(suptitle, fontsize=18, y=suptitle_pos)\n", + " return plt\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# non-stacked bar chart, for only plotting exec time or total time\n", + "def plotOneTestColoredTicks(df_data, suptitle=\"title\", suptitle_pos=1.00, subtitle=\"subchart\", subtitle_size='medium', highlight_ticks=None):\n", + " #f = plt.figure(figsize=[14, 8])\n", + " #plt.figure(figsize=[14, 8])\n", + " plt.style.use('ggplot')\n", + " plt.figure()\n", + " df_total_times = df_data[['exec_time', 'parse_time']].copy()\n", + " df_total_times.fillna(0, inplace=True)\n", + " df_total_times['time'] = df_total_times['exec_time'] + df_total_times['parse_time']\n", + " ax = df_total_times.sort_values('time')[['time']].plot.bar(figsize=(14,8), color=COLORS_DEFAULT['blue'])\n", + " # fontsize = {size in points, 'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n", + " ax.set_title(subtitle, fontsize=subtitle_size)\n", + " ax.set(ylabel=\"seconds\", xlabel=\"\")\n", + " ax.xaxis.set_tick_params(labelrotation=60, labelsize=12)\n", + " if highlight_ticks is not None:\n", + " for highlight_tick in highlight_ticks:\n", + " for i in plt.gca().get_xticklabels():\n", + " if i.get_text() == highlight_tick:\n", + " i.set_color('tab:green')\n", + " i.set_fontstyle('oblique')\n", + " i.set_fontweight('heavy')\n", + "\n", + " # {'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan'}\n", + " #[i.set_color('tab:green') for i in plt.gca().get_xticklabels() if i.get_text() == highlight_tick]\n", + " adjust_text_labels(labelBarHeights(ax))\n", + " plt.suptitle(suptitle, fontsize=18, y=suptitle_pos)\n", + " return plt\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotOneTestGrouped(df_1, test_title=\"test_title\", title=\"plot Title\", colors=['tab:blue', 'tab:orange'], sort_by='websnark-bn128-two-pairings', group_order=None):\n", + " # to group by engine, pivot to `columns='engine'`\n", + " #df_1 = pd.pivot_table(df_1, values='exec_time', columns=['engine'], index=['bench_name'], aggfunc=np.mean)\n", + " # group by bench name\n", + " plt.style.use('ggplot')\n", + " df_1['time'] = df_1['exec_time'] + df_1['parse_time']\n", + " df_1 = pd.pivot_table(df_1, values='time', columns=['bench_name'], index=['engine'], aggfunc=np.mean)\n", + " # order to get websnark as the group on the left\n", + " if group_order is not None:\n", + " df_1 = df_1[group_order]\n", + "\n", + " plt.figure()\n", + " ax = df_1.sort_values(sort_by).plot.bar(figsize=(14,8), color=colors)\n", + " #ax = df_1.plot.bar(figsize=(14,8))\n", + " ax.set_title(test_title)\n", + " ax.set(ylabel='seconds', xlabel='')\n", + "\n", + " adjust_text_labels(labelBarHeights(ax))\n", + "\n", + " plt.suptitle(title, fontsize=16, y=0.98)\n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotOneTestGroupedExexTime(df_1, test_title=\"test_title\", title=\"plot Title\", colors=['tab:blue', 'tab:orange'], sort_by='websnark-bn128-two-pairings', group_order=None):\n", + " # to group by engine, pivot to `columns='engine'`\n", + " #df_1 = pd.pivot_table(df_1, values='exec_time', columns=['engine'], index=['bench_name'], aggfunc=np.mean)\n", + " # group by bench name\n", + " plt.style.use('ggplot')\n", + " df_1['time'] = df_1['exec_time']\n", + " df_1 = pd.pivot_table(df_1, values='time', columns=['bench_name'], index=['engine'], aggfunc=np.mean)\n", + " # order to get websnark as the group on the left\n", + " if group_order is not None:\n", + " df_1 = df_1[group_order]\n", + "\n", + " plt.figure()\n", + " ax = df_1.sort_values(sort_by).plot.bar(figsize=(14,8), color=colors)\n", + " #ax = df_1.plot.bar(figsize=(14,8))\n", + " ax.set_title(test_title)\n", + " ax.set(ylabel='seconds', xlabel='')\n", + "\n", + " adjust_text_labels(labelBarHeights(ax))\n", + "\n", + " plt.suptitle(title, fontsize=16, y=0.98)\n", + " return plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotScoutStackedTest(df_data, suptitle=\"title\", suptitle_pos=1.00, subtitle=\"subchart\", subtitle_size='medium'):\n", + " #plt.figure(figsize=[12,7])\n", + " plt.figure()\n", + " # for some reason, exec_time needs to come first for the stacked chart to display correctly\n", + " ax = df_data[['exec_time', 'parse_time']].sort_values('exec_time').plot.bar(stacked=True, color=[COLORS_DEFAULT['blue'], COLORS_DEFAULT['red']], figsize=(10,10))\n", + " # fontsize = {size in points, 'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n", + " ax.set_title(subtitle, fontsize=subtitle_size)\n", + " ax.set(ylabel=\"seconds\", xlabel=\"\")\n", + " df_total_times = df_data[['exec_time', 'parse_time']].copy()\n", + " df_total_times['total_time'] = df_total_times['exec_time'] + df_total_times['parse_time']\n", + " ax.xaxis.set_tick_params(labelrotation=60, labelsize=12)\n", + " adjust_text_labels(labelBarHeights(ax, lower_y_bound=False))\n", + " ax.legend(labels=[\"execution time\", \"startup time\"])\n", + " plt.suptitle(suptitle, fontsize=18, y=suptitle_pos)\n", + " #ax.autoscale(enable=True) \n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot biturbo benchmark: biturbo-token-eth1-mainnet-stateless-block-hexary-trie-keccak256-multiproof" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_biturbo_token = df_scout_data[df_scout_data['bench_name'] == 'biturbo-token-eth1-mainnet-stateless-block-hexary-trie-keccak256-multiproof']\n", + "\n", + "df_scout_means_biturbo_token = df_scout_biturbo_token.groupby(['engine']).mean()\n", + "df_scout_fast_biturbo_token = filterDfEngines(df_scout_biturbo_token, ['v8-interpreter'])\n", + "df_scout_fast_biturbo_means = df_scout_fast_biturbo_token.groupby(['engine']).mean()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTest(df_scout_means_biturbo_token,\n", + " suptitle=\"\\nbiturbo token - all scout engines\",\n", + " suptitle_pos=1.05,\n", + " subtitle=\"biturbo-token-eth1-mainnet-stateless-block-hexary-trie-keccak256-multiproof\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-biturbo-token-all-engines.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_fast_biturbo_means,\n", + " suptitle=\"\\nbiturbo token - fast scout engines - wasm compilers vs interpreters (v8 vs wabt)\",\n", + " suptitle_pos=1.03,\n", + " subtitle=\"biturbo-token-eth1-mainnet-stateless-block-hexary-trie-keccak256-multiproof\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-biturbo-token-compilers-vs-interpreters.png', bbox_inches='tight')\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot bignum benchmark: ecrecover-eth1-txns-websnark-secp256k1-verify-72-sigs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_ecrecover_websnark_secp256k1 = df_scout_data[df_scout_data['bench_name'] == 'ecrecover-eth1-txns-websnark-secp256k1-verify-72-sigs']\n", + "\n", + "df_scout_means_ecrecover_websnark_secp256k1 = df_scout_ecrecover_websnark_secp256k1.groupby(['engine']).mean()\n", + "df_scout_fast_ecrecover = filterDfEngines(df_scout_ecrecover_websnark_secp256k1,\n", + " ['v8-interpreter', 'scoutcpp-wabt-no-bignums', 'wabt-no-bignums'])\n", + "df_scout_fast_ecrecover_means = df_scout_fast_ecrecover.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTest(df_scout_means_ecrecover_websnark_secp256k1,\n", + " suptitle=\"websnark-secp256k1-sig-verify - all Scout engines\",\n", + " suptitle_pos=1.0,\n", + " subtitle=\"ecrecover-eth1-txns-websnark-secp256k1-verify-72-sigs\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "\n", + "fig_plt.savefig('../images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-all-engines.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_fast_ecrecover_means,\n", + " #suptitle=\"websnark-secp256k1-sig-verify - fast Scout engines - compilers (v8) vs interpreters (wabt)\",\n", + " suptitle=\"compiler engines - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n interpreter engine (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"ecrecover-eth1-txns-websnark-secp256k1-verify-72-sigs\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-wabt-vs-v8.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot bignum benchmark: ecpairing-zkrollup-websnark-bn128-two-pairings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_ecpairing_zkrollup_bn128_pairings = df_scout_data[df_scout_data['bench_name'] == 'ecpairing-zkrollup-websnark-bn128-two-pairings']\n", + "\n", + "df_scout_means_ecpairing_zkrollup = df_scout_ecpairing_zkrollup_bn128_pairings.groupby(['engine']).mean()\n", + "df_scout_fast_ecpairing_zkrollup = filterDfEngines(df_scout_ecpairing_zkrollup_bn128_pairings,\n", + " ['v8-interpreter', 'scoutcpp-wabt-no-bignums', 'wabt-no-bignums'])\n", + "df_scout_fast_means_ecpairing_zkrollup = df_scout_fast_ecpairing_zkrollup.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTest(df_scout_means_ecpairing_zkrollup,\n", + " suptitle=\"websnark-bn128-pairings - all Scout engines\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-all-engines.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_fast_means_ecpairing_zkrollup,\n", + " suptitle=\"compiler engines - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n interpreter engine (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large')\n", + "fig_plt.savefig('../images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-v8.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## start with rust bn128 pairings, compare to native, and compare interpreters to compilers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rolluprs_bn128_pairings = df_scout_data[df_scout_data['bench_name'] == 'ecpairing-zkrollup-rust-wasm-bn128-two-pairings']\n", + "\n", + "df_scout_fast_rolluprs = filterDfEngines(df_scout_rolluprs_bn128_pairings,\n", + " ['scoutcpp-wabt-no-bignums', 'wabt-no-bignums'])\n", + "\n", + "df_scout_means_rolluprs = df_scout_rolluprs_bn128_pairings.groupby(['engine']).mean()\n", + "\n", + "#df_scout_fast_means_rolluprs = df_scout_fast_rolluprs.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTest(df_scout_means_rolluprs,\n", + " suptitle=\"rollup.rs-bn128-pairings - all Scout engines\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-all-engines.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## websnark bn128 pairings versus rust->wasm bn128 pairings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### add rust-native to rust-wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rolluprs_bn128_pairings = df_scout_data[df_scout_data['bench_name'].isin(\n", + " ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + ")]\n", + "\n", + "#df_scout_rolluprs_rustnative = df_scout_rolluprs_rustnative.fillna(0)\n", + "\n", + "df_scout_means_rolluprs = df_scout_rolluprs_bn128_pairings.groupby(['engine']).mean()\n", + "df_scout_fast_rolluprs = filterDfEngines(df_scout_rolluprs_bn128_pairings,\n", + " ['v8-interpreter', 'scoutcpp-wabt-no-bignums', 'wabt-no-bignums'])\n", + "df_scout_fast_means_rolluprs = df_scout_fast_rolluprs.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_fast_means_rolluprs,\n", + " suptitle=\"compiler engines - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n interpreter engine (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large')\n", + "fig_plt.savefig('../images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-v8.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# rust vs websnark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_vs_websnark = df_scout_data[\n", + " df_scout_data['bench_name'].isin(\n", + " ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-websnark-bn128-two-pairings']\n", + " )\n", + "].copy()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_scout_rust_vs_websnark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_scout_rust_vs_websnark.replace('ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'rust-wasm-bn128-two-pairings', inplace=True)\n", + "#df_scout_rust_vs_websnark.replace('ecpairing-zkrollup-websnark-bn128-two-pairings', 'websnark-bn128-two-pairings', inplace=True)\n", + "\n", + "df_scout_rust_vs_websnark.replace('ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'rust-wasm', inplace=True)\n", + "df_scout_rust_vs_websnark.replace('ecpairing-zkrollup-websnark-bn128-two-pairings', 'websnark-wasm', inplace=True)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_vs_websnark = df_scout_rust_vs_websnark[df_scout_rust_vs_websnark['engine'].isin(\n", + "['wabt-superops-bignums-slowmont', 'wabt-bignums-fasthost', 'wabt-superops-bignums', 'rust-native', 'v8-liftoff', 'v8-turbofan'] \n", + ")].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_scout_rust_vs_websnark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# {'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple', 'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan'}\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestGrouped(df_scout_rust_vs_websnark,\n", + " title=\"websnark vs rust - zkrollup bn128 pairings (websnark/bn128 vs rollup.rs)\",\n", + " test_title=\"\",\n", + " group_order=['websnark-wasm', 'rust-wasm'],\n", + " sort_by=\"rust-wasm\",\n", + " colors=['tab:blue', 'tab:orange'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt.savefig('../images/websnark-vs-rust-wasm-zkrollup-bn128-pairings.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_rolluprs_native = df_scout_data[df_scout_data['bench_name'] == 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "df_rolluprs_native = df_rolluprs_native.fillna(0)\n", + "\n", + "df_rolluprs_native.replace('ecpairing-zkrollup-rust-native-bn128-two-pairings', 'rust-native', inplace=True)\n", + "\n", + "\n", + "df_scout_rust_vs_websnark_fast_vs_native = df_scout_rust_vs_websnark.append(df_rolluprs_native, sort=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "fig_plt = plotOneTestGrouped(df_scout_rust_vs_websnark_fast_vs_native,\n", + " title=\"websnark vs rust - zkrollup bn128 pairings (websnark/bn128 vs rollup.rs)\",\n", + " test_title=\"\",\n", + " sort_by=[\"rust-native\", \"rust-wasm\", \"websnark-wasm\"],\n", + " colors=['tab:green', 'tab:blue', 'tab:orange'],\n", + " group_order=[\"rust-native\", \"websnark-wasm\", \"rust-wasm\"])\n", + "\n", + "fig_plt.savefig('../images/websnark-vs-rust-wasm-native-zkrollup-bn128-pairings', bbox_inches='tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### TODO: adjust bar size and placement?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Just websnark against native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# print out the engine names\n", + "\n", + "df_websnark_vs_native = df_scout_data[df_scout_data['bench_name'] == 'ecpairing-zkrollup-websnark-bn128-two-pairings'].copy()\n", + "df_websnark_vs_native.replace('ecpairing-zkrollup-websnark-bn128-two-pairings', 'websnark-bn128-two-pairings', inplace=True)\n", + "\n", + "df_websnark_vs_native['engine'].unique().tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_websnark_vs_native = df_scout_data[df_scout_data['bench_name'] == 'ecpairing-zkrollup-websnark-bn128-two-pairings'].copy()\n", + "df_websnark_vs_native.replace('ecpairing-zkrollup-websnark-bn128-two-pairings', 'websnark-bn128-two-pairings', inplace=True)\n", + "\n", + "df_websnark_vs_native = df_websnark_vs_native[df_websnark_vs_native['engine'].isin(\n", + "['wabt-bignums-fasthost', 'wabt-bignums-slowhost', 'wabt-superops-bignums', 'wabt-superops-bignums-slowmont', 'wabt-with-bignums-and-superops', 'wabt-with-bignums', 'rust-native', 'v8-liftoff', 'v8-turbofan'] \n", + ")]\n", + "\n", + "df_websnark_vs_native = df_websnark_vs_native.append(df_rolluprs_native)\n", + "\n", + "df_websnark_vs_native_means = df_websnark_vs_native.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_websnark_vs_native_means" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_websnark_vs_native_means,\n", + " suptitle=\"compiler engines - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n interpreter engine (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"bn128-two-pairings - websnark vs rust-native\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/websnark-vs-native-bn128-two-pairings-v8.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Try Seaborn with error bars" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_websnark_vs_native['time'] = df_websnark_vs_native['exec_time'] + df_websnark_vs_native['parse_time']\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "\n", + "ax = sns.barplot(x='engine', y='time', data=df_websnark_vs_native)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "\n", + "ax = sns.barplot(x='engine', y='exec_time', hue='bench_name', data=df_scout_rust_vs_websnark)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot bignum benchmark: daiquiri-zkmixer-websnark-bn128-groth16-four-pairings-and-mimc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_daiquiri_zkmixer = df_scout_data[df_scout_data['bench_name'] == 'daiquiri-zkmixer-websnark-bn128-groth16-four-pairings-and-mimc']\n", + "\n", + "df_scout_means_daiquiri_zkmixer = df_scout_daiquiri_zkmixer.groupby(['engine']).mean()\n", + "df_scout_fast_daiquiri_zkmixer = filterDfEngines(df_scout_daiquiri_zkmixer,\n", + " ['v8-interpreter', 'scoutcpp-wabt-no-bignums', 'wabt-no-bignums'])\n", + "df_scout_fast_means_daiquiri_zkmixer = df_scout_fast_daiquiri_zkmixer.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plotOneTest(df_scout_means_daiquiri_zkmixer,\n", + " suptitle=\"daiquiri-zkmixer - all Scout engines\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"daiquiri-zkmixer-websnark-bn128-groth16-four-pairings-and-mimc\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-daiquiri-all-engines.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_fast_means_daiquiri_zkmixer,\n", + " suptitle=\"wasm compilers - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n wasm interpreter (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"daiquiri-zkmixer-websnark-bn128-groth16-four-pairings-and-mimc\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/scout-daiquiri-v8-vs-wabt.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import geth precompile results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_precompiles = read_results(join(CSV_RESULT_DIR, GETH_PRECOMPILE_RESULT_FILE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_geth_precompiles" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def splitName(test_name):\n", + " ix = test_name.find(\"/\")\n", + " return (test_name[0:ix], test_name[ix+1:])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_precompiles['precompile_name'], df_geth_precompiles['input_name'] = zip(*df_geth_precompiles.apply(lambda row: splitName(row['test_name']), axis=1))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_precompiles = df_geth_precompiles[['precompile_name', 'input_name', 'gas', 'time']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing = df_geth_precompiles[df_geth_precompiles[\"input_name\"] == \"two_point_match_rollup_input\"].copy()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import standalone wasm results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wasm = read_results(join(CSV_RESULT_DIR, STANDALONE_WASM_RESULT_FILE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import native results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_native_raw = read_results(join(CSV_RESULT_DIR, NATIVE_RESULT_FILE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "native_results = {}\n", + "for i in range(len(df_native_raw)):\n", + " test_name = df_native_raw['test_name'][i]\n", + " native_results[test_name] = df_native_raw['elapsed_times'][i].split(', ')\n", + "\n", + " \n", + "test_names = df_native_raw['test_name'].tolist()\n", + "\n", + "native_times = {}\n", + "for test in test_names:\n", + " avg = np.mean([float(t) for t in native_results[test]])\n", + " native_times[test] = avg\n", + "\n", + " \n", + "df_native = pd.DataFrame.from_dict(native_times, orient='index', columns=['elapsed_time'])\n", + "df_native['engine'] = 'rust-native'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### sorted lists of test names for plotting inputs in order" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "blake2b_test_names = [name for name in df_wasm['test_name'].unique().tolist() if 'blake2b' in name]\n", + "blake2b_test_names.sort()\n", + "blake2b_test_names_desc = blake2b_test_names.copy()\n", + "blake2b_test_names_desc.reverse()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sha1_test_names = [name for name in df_wasm['test_name'].unique().tolist() if 'sha1' in name]\n", + "sha1_test_names.sort()\n", + "sha1_test_names_desc = sha1_test_names.copy()\n", + "sha1_test_names_desc.reverse()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "bn128_mul_test_names = ['bn128_mul-cdetrio11', 'bn128_mul-chfast2', 'bn128_mul-chfast1']\n", + "bn128_add_test_names = ['bn128_add-cdetrio11', 'bn128_add-cdetrio14', 'bn128_add-cdetrio10']\n", + "bn128_pairing_test_names = ['bn128_pairing-ten_point_match_1', 'bn128_pairing-two_point_match_2', 'bn128_pairing-one_point']\n", + "\n", + "bn128_mul_test_names.reverse()\n", + "bn128_add_test_names.reverse()\n", + "bn128_pairing_test_names.reverse()\n", + "\n", + "bn128_mul_test_names_desc = ['bn128_mul-cdetrio11', 'bn128_mul-chfast2', 'bn128_mul-chfast1']\n", + "bn128_add_test_names_desc = ['bn128_add-cdetrio11', 'bn128_add-cdetrio14', 'bn128_add-cdetrio10']\n", + "bn128_pairing_test_names_desc = ['bn128_pairing-ten_point_match_1', 'bn128_pairing-two_point_match_2', 'bn128_pairing-one_point']\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bls12_test_names = ['bls12-381-aggreg-32-pubkeys-verify-sig', 'bls12-381-aggreg-64-pubkeys-verify-sig', 'bls12-381-aggreg-128-pubkeys-verify-sig']\n", + "bls12_test_names_desc = bls12_test_names.copy()\n", + "bls12_test_names_desc.reverse()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "modexp_test_names = [name for name in df_wasm['test_name'].unique().tolist() if 'modexp' in name]\n", + "modexp_test_names.sort()\n", + "modexp_test_names_desc = modexp_test_names.copy()\n", + "modexp_test_names_desc.reverse()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_tests = sha1_test_names + blake2b_test_names + modexp_test_names + ['ed25519-verify-ten-inputs'] + bls12_test_names \\\n", + " + bn128_add_test_names + bn128_mul_test_names + bn128_pairing_test_names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Prepare interpreter dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_interp_test_names = []\n", + "\n", + "interp_results_for_df = []\n", + "for engine in INTERPRETER_ENGINES:\n", + " df_engine = df_wasm[df_wasm['engine'] == engine]\n", + " df_means = df_engine.groupby(['test_name']).mean()\n", + " test_names = df_engine['test_name'].unique().tolist()\n", + " all_interp_test_names.extend(test_names)\n", + " for test_name in test_names:\n", + " interp_results_for_df.append([engine, test_name] + df_means.loc[test_name].tolist())\n", + "\n", + " \n", + "all_interp_test_names = set(all_interp_test_names)\n", + "\n", + "#interp_results_for_df\n", + "\n", + "df_interp = pd.DataFrame(interp_results_for_df)\n", + "df_interp.columns = ['engine', 'test_name', 'elapsed_time', 'parse_time', 'exec_time']\n", + "df_interp.set_index('engine', inplace=True)\n", + "df_interp['total_time'] = df_interp['parse_time'] + df_interp['exec_time']\n", + "#df_interp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wasm['test_name'].unique().tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpreter engines compared" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotInterpThreeTests(df_testdata, three_names, title=\"Title\", filter_engines=None):\n", + " if filter_engines is not None:\n", + " df_testdata = filterDfEngines(df_testdata.reset_index(), filter_engines)\n", + " df_testdata.set_index('engine', inplace=True)\n", + " df_1 = df_testdata[df_testdata['test_name'] == three_names[0]]\n", + " df_2 = df_testdata[df_testdata['test_name'] == three_names[1]]\n", + " df_3 = df_testdata[df_testdata['test_name'] == three_names[2]]\n", + " plt.figure()\n", + " f, (ax1) = plt.subplots(1, 3, sharey=True, figsize=(16,7))\n", + " df_1.sort_values('total_time')[['parse_time', 'exec_time']].plot.bar(ax=ax1[0], stacked=True)\n", + " ax1[0].set_title(three_names[0])\n", + " ax1[0].set(xlabel='')\n", + " df_2.sort_values('total_time')[['parse_time', 'exec_time']].plot.bar(ax=ax1[1], stacked=True)\n", + " ax1[1].set_title(three_names[1])\n", + " ax1[1].set(xlabel='')\n", + " df_3.sort_values('total_time')[['parse_time', 'exec_time']].plot.bar(ax=ax1[2], stacked=True)\n", + " ax1[2].set_title(three_names[2])\n", + " ax1[2].set(xlabel='')\n", + " adjust_text_labels(labelBarHeights(ax1[0]), ax=ax1[0])\n", + " adjust_text_labels(labelBarHeights(ax1[1]), ax=ax1[1])\n", + " adjust_text_labels(labelBarHeights(ax1[2]), ax=ax1[2])\n", + " ax1[0].legend(labels=[\"startup time\", \"execution time\"])\n", + " ax1[1].legend(labels=[\"startup time\", \"execution time\"])\n", + " ax1[2].legend(labels=[\"startup time\", \"execution time\"])\n", + "\n", + " ax1[1].tick_params(axis='y', left=True, labelleft=True)\n", + " ax1[2].tick_params(axis='y', left=True, labelleft=True)\n", + " \n", + " scale_y = 1e-3\n", + " ticks_y = ticker.FuncFormatter(lambda y, pos: '{0:g}'.format(y/scale_y))\n", + " ax1[1].yaxis.set_major_formatter(ticks_y)\n", + " \n", + " ax1[0].set(ylabel='milliseconds')\n", + " \n", + " f.suptitle(title, fontsize=16, y=0.98)\n", + " return plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpThreeTests(df_interp, blake2b_test_names, title=\"wasm interpreters compared - blake2b\")\n", + "plt_fig.savefig('../images/wasm-interp-blake2b.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interpreters - Execution time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotThreeTestsExecTime(df_testdata, three_names, title=\"Title\", filter_engines=None):\n", + " if filter_engines is not None:\n", + " df_testdata = filterDfEngines(df_testdata.reset_index(), filter_engines)\n", + " df_testdata.set_index('engine', inplace=True)\n", + " df_1 = df_testdata[df_testdata['test_name'] == three_names[0]]\n", + " df_2 = df_testdata[df_testdata['test_name'] == three_names[1]]\n", + " df_3 = df_testdata[df_testdata['test_name'] == three_names[2]]\n", + " plt.figure()\n", + " f, (ax1) = plt.subplots(1, 3, sharey=True, figsize=(16,7))\n", + " df_1.sort_values('exec_time')[['exec_time']].plot.bar(ax=ax1[0], color=COLORS_DEFAULT['blue'])\n", + " ax1[0].set_title(three_names[0])\n", + " ax1[0].set(xlabel='')\n", + " df_2.sort_values('exec_time')[['exec_time']].plot.bar(ax=ax1[1], color=COLORS_DEFAULT['blue'])\n", + " ax1[1].set_title(three_names[1])\n", + " ax1[1].set(xlabel='')\n", + " df_3.sort_values('exec_time')[['exec_time']].plot.bar(ax=ax1[2], color=COLORS_DEFAULT['blue'])\n", + " ax1[2].set_title(three_names[2])\n", + " ax1[2].set(xlabel='')\n", + " adjust_text_labels(labelBarHeights(ax1[0]), ax=ax1[0])\n", + " adjust_text_labels(labelBarHeights(ax1[1]), ax=ax1[1])\n", + " adjust_text_labels(labelBarHeights(ax1[2]), ax=ax1[2])\n", + " ax1[0].legend(labels=[\"execution time\"])\n", + " ax1[1].legend(labels=[\"execution time\"])\n", + " ax1[2].legend(labels=[\"execution time\"])\n", + "\n", + " ax1[1].tick_params(axis='y', left=True, labelleft=True)\n", + " ax1[2].tick_params(axis='y', left=True, labelleft=True)\n", + " \n", + " scale_y = 1e-3\n", + " ticks_y = ticker.FuncFormatter(lambda y, pos: '{0:g}'.format(y/scale_y))\n", + " ax1[1].yaxis.set_major_formatter(ticks_y)\n", + " \n", + " ax1[0].set(ylabel='milliseconds')\n", + " \n", + " f.suptitle(title, fontsize=16, y=0.98)\n", + " return f\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotThreeTestsExecTime(df_interp, blake2b_test_names, title=\"wasm interpreters compared - blake2b\")\n", + "plt_fig.savefig('../images/wasm-interp-blake2b-exec.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chart of only the fast interpreters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpThreeTests(df_interp,\n", + " blake2b_test_names,\n", + " title=\"wasm interpreters compared - blake2b\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/wasm-interp-blake2b-fast.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpThreeTests(df_interp,\n", + " modexp_test_names,\n", + " title=\"wasm interpreters compared - modexp\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/wasm-interp-modexp.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpThreeTests(df_interp,\n", + " bn128_add_test_names,\n", + " title=\"wasm interpreters compared - bn128_add\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/wasm-interp-bn128-add.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotThreeTestsExecTime(df_interp,\n", + " blake2b_test_names,\n", + " title=\"wasm interpreters compared - blake2b\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/wasm-interp-blake2b-life-wagon-v8.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotThreeTestsExecTime(df_interp,\n", + " bls12_test_names,\n", + " title=\"wasm interpreters compared - bls12\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/wasm-interp-bls12.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compiler Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "compiler_results_for_df = []\n", + "for engine in COMPILER_ENGINES:\n", + " df_engine = df_wasm[df_wasm['engine'] == engine]\n", + " df_means = df_engine.groupby(['test_name']).mean()\n", + " test_names = df_engine['test_name'].unique().tolist()\n", + " for test_name in test_names:\n", + " compiler_results_for_df.append([engine, test_name] + df_means.loc[test_name].tolist())\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_compiler = pd.DataFrame(compiler_results_for_df)\n", + "\n", + "df_compiler.columns = ['engine', 'test_name', 'elapsed_time', 'compile_time', 'exec_time']\n", + "df_compiler.set_index('engine', inplace=True)\n", + "df_compiler['total_time'] = df_compiler['compile_time'] + df_compiler['exec_time']\n", + "\n", + "#df_compiler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## JIT bomb chart - guido-fuzzer-find-1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_compiler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotJitBomb(df_jit, title=\"jit bomb chart\", log_scale=False):\n", + " f = plt.figure(figsize=(14,8))\n", + "\n", + " bombplot = sns.barplot(x='engine', y='value', hue='exec_or_compile', dodge=True, data=df_jit)\n", + " if log_scale:\n", + " bombplot.set(yscale=\"log\")\n", + "\n", + " # annotating bars: https://github.com/mwaskom/seaborn/issues/1582\n", + " for p in bombplot.patches:\n", + " bombplot.annotate(\n", + " #format(p.get_height(), '.4f'),\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + " f.suptitle(title, fontsize=18, y=0.95)\n", + " return f" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_jitbomb_1 = df_compiler[df_compiler['test_name'] == 'guido-fuzzer-find-1'].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_jitbomb_1.drop(['elapsed_time'], axis=1, inplace=True)\n", + "df_jitbomb_1.drop(['total_time'], axis=1, inplace=True)\n", + "df_jitbomb_1.reset_index(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_jitbomb_1 = df_jitbomb_1.melt(id_vars=[\"engine\", \"test_name\"], var_name=\"exec_or_compile\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotJitBomb(df_jitbomb_1, \"JIT bomb guido-fuzzer-find-1\")\n", + "fig_plt.savefig('../images/standalone-jitbomb-guido-fuzzer-find-1.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotJitBomb(df_jitbomb_1, \"JIT bomb guido-fuzzer-find-1 - log scale\", log_scale=True)\n", + "fig_plt.savefig('../images/standalone-jitbomb-guido-fuzzer-find-1-log-scale.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### guido-fuzzer-find-2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_jitbomb_2 = df_compiler[df_compiler['test_name'] == 'guido-fuzzer-find-2'].copy()\n", + "\n", + "df_jitbomb_2.drop(['elapsed_time'],axis=1,inplace=True)\n", + "df_jitbomb_2.drop(['total_time'],axis=1,inplace=True)\n", + "df_jitbomb_2.reset_index(inplace=True)\n", + "\n", + "df_jitbomb_2 = df_jitbomb_2.melt(id_vars=[\"engine\", \"test_name\"], var_name=\"exec_or_compile\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotJitBomb(df_jitbomb_2, \"JIT bomb guido-fuzzer-find-2 (log scale)\", log_scale=True)\n", + "\n", + "fig_plt.savefig('../images/standalone-jitbomb-guido-fuzzer-find-2-log-scale.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# TODO: mention v8-liftoff version here\n", + "# TODO: try a later v8-liftoff version?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compiler function\n", + "\n", + "*Note: Fix scale for v8-turbofan execution time*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotCompilerStackedOneTest(df_benchdata, test_name):\n", + " plt.figure()\n", + " df_1 = df_benchdata[df_benchdata['test_name'] == test_name]\n", + " # for some reason, exec_time needs to come first for the stacked chart to display correctly\n", + " ax = df_1[['exec_time', 'compile_time']].sort_values('exec_time').plot.bar(stacked=True, color=[COLORS_DEFAULT['blue'], COLORS_DEFAULT['red']], figsize=(10,10))\n", + " ax.set_title(\"Compilers - {}\".format(test_name))\n", + " df_total_times = df_1[['exec_time', 'compile_time']].copy()\n", + " df_total_times['total_time'] = df_total_times['exec_time'] + df_total_times['compile_time']\n", + " ymax = max(df_total_times[['total_time']].max()) * 1.3 # 30% larger for padding between top of bars and top of graph\n", + " ymin = min([x for x in df_1[['exec_time', 'compile_time']].min().tolist() if x > 0])\n", + " ymin = ymin * 0.8 # 0.8 to get a number 20% smaller, fix to make bar appear for smallest exec time\n", + " ax.set_ylim(ymin, ymax)\n", + " ax.set_yscale(\"log\")\n", + " plt.ylabel(\"seconds (log scale)\")\n", + " adjust_text_labels(labelBarHeights(ax, lower_y_bound=False))\n", + " ax.legend(labels=[\"execution time\", \"compile time\"])\n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotCompilerStackedOneTest(df_compiler, \"blake2b-8415-bytes\")\n", + "fig_plt.savefig('../images/standalone-blake2b-8415-bytes-compilers.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Add rust-native to compiler engines chart" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge df_native and df_compiler into one dataframe\n", + "# both dataframes must have same columns to merge them:\n", + "# engine, test_name, elapsed_time, compile_time, exec_time, total_time\n", + "\n", + "df_native_merge = df_native.copy()\n", + "df_native_merge.reset_index(inplace=True)\n", + "df_native_merge.columns = ['test_name', 'elapsed_time', 'engine']\n", + "df_native_merge['compile_time'] = 0\n", + "df_native_merge['exec_time'] = df_native_merge['elapsed_time']\n", + "df_native_merge['total_time'] = df_native_merge['elapsed_time']\n", + "df_native_and_compile = pd.concat([df_compiler.reset_index(), df_native_merge], sort=False)\n", + "df_native_and_compile.reset_index(drop=True, inplace=True)\n", + "df_native_and_compile.set_index('engine', inplace=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotCompilerStackedOneTest(df_native_and_compile, \"bls12-381-aggreg-128-pubkeys-verify-sig\")\n", + "fig_plt.savefig('../images/standalone-bls12-381-aggreg-128-pubkeys-verify-sig-compilers-vs-native.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpreter vs Compiler speedup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# merge df_compiler and df_interp\n", + "df_interp_merge = df_interp.copy()\n", + "df_interp_merge.columns = ['test_name', 'elapsed_time', 'compile_time', 'exec_time', 'total_time']\n", + "df_interp_and_compile = pd.concat([df_interp_merge, df_compiler])\n", + "df_interp_and_compile = df_interp_and_compile.reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def add_engine_ratio_col(df_two_engines, interp_engine, compiler_engine):\n", + " df_two = df_two_engines.copy(deep=True)\n", + " df_interp = df_two[df_two['engine'] == interp_engine].copy()\n", + " df_compile = df_two[df_two['engine'] == compiler_engine]\n", + " for index, row in df_interp.iterrows():\n", + " test_name = row['test_name']\n", + " row_compiler = df_compile[df_compile['test_name'] == test_name]\n", + " if len(row_compiler) != 1:\n", + " print(\"ERROR. SHOULD HAVE 1 MATCH ONLY\")\n", + " break\n", + " compiler_exec_time = row_compiler.iloc[0]['exec_time']\n", + " exec_ratio = row['exec_time'] / compiler_exec_time\n", + " df_interp.at[index, 'compiler_exec_time'] = compiler_exec_time\n", + " df_interp.at[index, 'interp_slowdown'] = exec_ratio\n", + "\n", + " return df_interp" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wabt_v8liftoff = add_engine_ratio_col(df_interp_and_compile, \"wabt\", \"v8-liftoff\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotCompilerSpeedup(df_compiler_speedup, interp_name=\"interp\", compiler_name=\"compiler\"):\n", + " df_is = df_compiler_speedup.copy()\n", + " plot_tests = df_is['test_name'].tolist()\n", + " test_index = all_tests.copy()\n", + " for test_name in all_tests:\n", + " if test_name not in plot_tests:\n", + " test_index.remove(test_name)\n", + " \n", + " df_is = df_is.set_index('test_name')\n", + " df_is = df_is.reindex(test_index)\n", + " plt.figure()\n", + " ax = df_is['interp_slowdown'].plot.bar(figsize=(13,7), color=COLORS_DEFAULT['blue'])\n", + " ax.set_title(\"{} (Interpreter) vs {} (Compiler) - speedup\".format(interp_name, compiler_name))\n", + " ax.set(ylabel=\"compiler speedup (log scale)\", xlabel=\"\")\n", + "\n", + " y_max = ax.get_ylim()[1]\n", + " y_max = y_max * 5 # padding between top of bars and legend\n", + " ax.set_yscale('log')\n", + " ax.set_ylim(0.1, y_max) # adjustment to move horizontal line higher on chart\n", + "\n", + " adjust_text_labels(labelBarHeights(ax, to_ms=False, to_factor=1, lower_y_bound=False))\n", + "\n", + " plt.axhline(y=1.0, linewidth=2, ls='--', color='r')\n", + "\n", + " handles, _ = ax.get_legend_handles_labels()\n", + "\n", + " plt.legend(labels=[\"compiler engine is faster (ratio > 1x)\", \"compiler speedup \\n (ratio: interp_exec_time/compiler_exec_time)\"])\n", + "\n", + " #f.suptitle(title, fontsize=16, y=0.95)\n", + " return plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotCompilerSpeedup(df_wabt_v8liftoff, interp_name=\"wabt\", compiler_name=\"v8-liftoff\")\n", + "plt_fig.savefig('../images/standalone-wabt-vs-v8-liftoff-speedup.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## All precompiles compared (are interpreters feasible?)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotInterpOneEngine(df_benchdata, engine, test_names, title=\"\"):\n", + " df_bench = df_benchdata.loc[engine][df_benchdata.loc[engine]['test_name'].isin(test_names)]\n", + " df_bench.set_index('test_name', inplace=True)\n", + " df_bench = df_bench.reindex(test_names)\n", + " plt.figure()\n", + " ax = df_bench[['parse_time', 'exec_time']].plot.bar(stacked=True, figsize=(15,7), color=[COLORS_DEFAULT['red'], COLORS_DEFAULT['blue']])\n", + " plt.ylabel(\"seconds\")\n", + " ax.set(xlabel='')\n", + " # doing get_legend_handles_labels() so we can control the order of the labels in the legend \n", + " handles, labels = ax.get_legend_handles_labels()\n", + " plt.legend([handles[1], handles[0]], ['execution time (top bar/label)', 'startup time (bottom bar/label - bar not visible for times < 20.0ms)'])\n", + " adjust_text_labels(labelBarHeights(ax))\n", + " plt.suptitle(title, fontsize=16, y=0.95)\n", + " return plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpOneEngine(df_interp, 'wasmi', all_tests, \"Wasmi - all Precompiles (existing and proposed) compared\")\n", + "plt_fig.savefig('../images/standalone-wasmi-all-precompiles.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotInterpOneEngine(df_interp, 'wabt', all_tests, \"Wabt - all Precompiles (existing and proposed) compared\")\n", + "plt_fig.savefig('../images/standalone-wabt-all-precompiles.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt_fig = plotThreeTestsExecTime(df_interp,\n", + " bn128_pairing_test_names,\n", + " title=\"wasm interpreters compared - bn128 pairings\",\n", + " filter_engines=[\"life\", \"wagon\", \"v8-interpreter\"])\n", + "plt_fig.savefig('../images/standalone-wasm-interp-bn128-pairings.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Are interpreter engines feasible for bn128 pairings?\n", + "TODO: highlight pairing two point match in interpreter engines\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotCompilerStackedOneTest(df_native_and_compile, \"bn128_pairing-two_point_match_2\")\n", + "\n", + "fig_plt.savefig('../images/standalone-rust-bn128-pairings-compiler-vs-native.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Is any kind of wasm engine, interpreters or compilers, feasible for bn128 pairings?\n", + "\n", + "V8-liftoff takes around 100ms, too slow relative to block processing time.\n", + "\n", + "Can it be sped up? From profiling, we'd see that the bottleneck is bignum arithmetic. Maybe \"native bignum methods\", such as a mul256 opcode, would bring a significant speedup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### add geth-native to df_scout_data for the rollup benchmark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing.loc[:, 'engine'] = \"go-native\"\n", + "df_geth_native_bn128pairing.loc[:, 'parse_time'] = 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing.rename(columns={'input_name':'bench_name','time':'exec_time'},inplace=True)\n", + "df_geth_native_bn128pairing.drop(['precompile_name'],axis=1,inplace=True)\n", + "df_geth_native_bn128pairing.drop(['gas'],axis=1,inplace=True)\n", + "df_geth_native_bn128pairing = df_geth_native_bn128pairing[['engine', 'bench_name', 'parse_time', 'exec_time']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing.loc[:,'bench_name'] = 'ecpairing-zkrollup-go-native-bn128-two-pairings'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_geth_native_bn128pairing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## list engine names" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "df_scout_rust_vs_websnark = df_scout_data[\n", + " df_scout_data['bench_name'].isin(\n", + " ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-websnark-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + " )\n", + "]\n", + "\n", + "df_scout_rust_vs_websnark = df_scout_rust_vs_websnark.append(df_geth_native_bn128pairing)\n", + "\n", + "df_scout_rust_vs_websnark['engine'].unique().tolist()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## function to prepare dataframe for rollup.rs and websnark plots\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def prepDfForRollupPlot(df_scout_data, df_geth_native, include_bench_names, only_include_engines=None, rename_engines=None):\n", + " df_scout_rollup = df_scout_data[\n", + " df_scout_data['bench_name'].isin(include_bench_names)\n", + " ].copy()\n", + "\n", + " df_scout_rollup = df_scout_rollup.fillna(0)\n", + " df_scout_rollup = df_scout_rollup.append(df_geth_native).copy()\n", + " if only_include_engines is not None:\n", + " df_scout_rollup = df_scout_rollup[df_scout_rollup['engine'].isin(only_include_engines)]\n", + " # rename_engines example: [['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums']]\n", + " if rename_engines is not None:\n", + " for rename in rename_engines:\n", + " df_scout_rollup.replace(rename[0], rename[1], inplace=True)\n", + " return df_scout_rollup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rust-wasm: bignums optimization - basic bignums speedup (wabt-with-bignums)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_bench_names = ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "df_scout_rolluprs_bignums_vs_nobignums = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan', 'wabt-bignums-slowhost-slowmont', 'wabt-no-bignums'],\n", + " rename_engines=[['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums']]\n", + " )\n", + "\n", + "df_scout_rolluprs_bignums_vs_nobignums = df_scout_rolluprs_bignums_vs_nobignums.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_rolluprs_bignums_vs_nobignums,\n", + " suptitle=\"rollup.rs-bn128-pairings - fast Scout engines (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-with-bignums\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-rollup-rs-bn128-pairings-with-vs-without-bignums.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Are interpreter engines feasible, given an API for fast bignums?\n", + "\n", + "From wabt-no-bignums at 1.18s, adding a bignum API achieves a 5x+ speedup to 172ms! Our interpreter is about 2x slower than a fast JIT engine, v8-liftoff at 91ms. Is there any way to speed it up?\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rust-wasm: remove wabt-no-bignums from chart. compare only wabt-with-bignums vs compilers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_bench_names = ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "\n", + "df_scout_rolluprs_bignums = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan', 'wabt-bignums-slowhost-slowmont'],\n", + " rename_engines=[['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums']]\n", + " )\n", + "\n", + "\n", + "df_scout_rolluprs_bignums = df_scout_rolluprs_bignums.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_rolluprs_bignums,\n", + " suptitle=\"rollup.rs-bn128-pairings - fast Scout engines (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-with-bignums\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-rollup-rs-bn128-pairings-wabt-with-bignums.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rust-wasm: Superops optimization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_bench_names = ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-bignums-slowhost-slowmont', 'wabt-bignums-slowhost-slowmont-superops']\n", + "\n", + "\n", + "df_scout_rollup_rustwasm_superops = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops']\n", + " ])\n", + "\n", + "df_scout_rollup_rustwasm_superops = df_scout_rollup_rustwasm_superops.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_rollup_rustwasm_superops,\n", + " suptitle=\"rollup.rs-bn128-pairings - fast Scout engines (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-bignums-superops\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-rollup-rs-bn128-pairings-superops.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### rust-wasm: in addition to superops, plot combied superops + fasthost, and superops + fasthost + fastmont" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_bench_names = ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-bignums-slowhost-slowmont', 'wabt-bignums-slowhost-slowmont-superops',\n", + " 'wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-fasthost-fastmont-superops']\n", + "\n", + "df_scout_rolluprs_all_combos = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops'],\n", + " ['wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-superops-fasthost-slowmont'],\n", + " ['wabt-bignums-fasthost-fastmont-superops', 'wabt-bignums-superops-fasthost-fastmont']\n", + " ])\n", + "\n", + "df_scout_rolluprs_all_combos = df_scout_rolluprs_all_combos.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_rolluprs_all_combos,\n", + " suptitle=\"rollup.rs-bn128-pairings - fast Scout engines (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-bignums-superops-fasthost-fastmont\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-rollup-rs-bn128-pairings-all-combos.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Optimization: websnark-wasm vs rust-wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_bench_names = ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'ecpairing-zkrollup-websnark-bn128-two-pairings']\n", + "\n", + "include_engines_list = ['v8-liftoff', 'v8-turbofan',\n", + " 'wabt-bignums-slowhost-slowmont', 'wabt-bignums-slowhost-slowmont-superops']\n", + "\n", + "df_scout_rust_vs_websnark = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['ecpairing-zkrollup-rust-wasm-bn128-two-pairings', 'rust-wasm'],\n", + " ['ecpairing-zkrollup-websnark-bn128-two-pairings', 'websnark-wasm'],\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops']\n", + " ])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestGrouped(df_scout_rust_vs_websnark,\n", + " title=\"websnark vs rust - zkrollup bn128 pairings (websnark/bn128 vs rollup.rs)\",\n", + " test_title=\"\",\n", + " group_order=['websnark-wasm', 'rust-wasm'],\n", + " sort_by=['rust-wasm'],\n", + " colors=['tab:orange', 'tab:blue'])\n", + "\n", + "\n", + "fig_plt.savefig('../images/bignums-zkrollup-rust-vs-websnark-bn128-pairings.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### drop rust-wasm, just plot websnark-wasm on different engines" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-bignums-slowhost-slowmont', 'wabt-bignums-slowhost-slowmont-superops']\n", + "\n", + "include_bench_names = ['ecpairing-zkrollup-websnark-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "df_scout_websnark = prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops']\n", + " ]\n", + " )\n", + "\n", + "df_scout_means_websnark = df_scout_websnark.groupby(['engine']).mean()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_means_websnark,\n", + " suptitle=\"websnark-bn128-pairings - engines compared (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-websnaark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-bignums-superops\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-websnark-bn128-pairings-superops.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### websnark-wasm optimization: \"fast-host\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-bignums-slowhost-slowmont', 'wabt-bignums-slowhost-slowmont-superops',\n", + " 'wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-fasthost-slowmont-superops']\n", + "\n", + "include_bench_names = ['ecpairing-zkrollup-websnark-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "df_scout_websnark_fasthost = prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops'],\n", + " ['wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-superops-fasthost']\n", + " ]\n", + " )\n", + "\n", + "df_scout_websnark_fasthost = df_scout_websnark_fasthost.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_websnark_fasthost,\n", + " suptitle=\"websnark-bn128-pairings - engines compared (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-bignums-superops-fasthost\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-websnark-bn128-pairings-fasthost.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### websnark-wasm optimization: interleaved montgomery" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan', 'wabt-bignums-slowhost-slowmont',\n", + " 'wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-slowhost-slowmont-superops',\n", + " 'wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-fasthost-fastmont-superops']\n", + "\n", + "include_bench_names = ['ecpairing-zkrollup-websnark-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "df_scout_websnark_fastmont = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops'],\n", + " ['wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-superops-fasthost'],\n", + " ['wabt-bignums-fasthost-fastmont-superops', 'wabt-bignums-superops-fasthost-fastmont']\n", + " ])\n", + "\n", + "df_scout_websnark_fastmont = df_scout_websnark_fastmont.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_websnark_fastmont,\n", + " suptitle=\"websnark-bn128-pairings - engines compared (v8-liftoff and wabt-with-bignums)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-websnaark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-bignums-superops-fasthost-fastmont\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-websnark-bn128-pairings-fastmont.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### websnark-wasm: Show startup time and execution time, for interleaved montgomery and other optimizations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_websnark_fastmont,\n", + " suptitle=\"wasm compilers - optimizing (v8-turbofan) and single-pass (v8-liftoff) \\n vs. \\n wasm interpreter (wabt) with bignum host funcs\",\n", + " suptitle_pos=1.07,\n", + " subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large')\n", + "\n", + "fig_plt.savefig('../images/bignums-websnark-bn128-pairings-startup-and-execution.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## plot v8 with bignums versus without\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['go-native', 'rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'v8-turbofan-with-bignums', 'v8-liftoff-with-bignums',\n", + " 'wabt-bignums-fasthost-fastmont-superops']\n", + "\n", + "include_bench_names = ['ecpairing-zkrollup-websnark-bn128-two-pairings',\n", + " 'ecpairing-zkrollup-rust-native-bn128-two-pairings']\n", + "\n", + "df_scout_v8_with_hostfuncs = prepDfForRollupPlot(df_scout_data,\n", + " df_geth_native_bn128pairing,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums'],\n", + " ['wabt-bignums-slowhost-slowmont-superops', 'wabt-bignums-superops'],\n", + " ['wabt-bignums-fasthost-slowmont-superops', 'wabt-bignums-superops-fasthost'],\n", + " ['wabt-bignums-fasthost-fastmont-superops', 'wabt-bignums-superops-fasthost-fastmont']\n", + " ]\n", + " )\n", + "\n", + "df_scout_v8_with_hostfuncs = df_scout_v8_with_hostfuncs.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.style.use('ggplot')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_v8_with_hostfuncs,\n", + " suptitle=\"websnark-bn128-pairings - engines compared (v8 without host funcs, v8 with host funcs)\",\n", + " suptitle_pos=1.02,\n", + " subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " subtitle_size='xx-large',\n", + " highlight_ticks=[\"v8-liftoff-with-bignums\", \"v8-turbofan-with-bignums\"])\n", + "\n", + "\n", + "\n", + "fig_plt.savefig('../images/bignums-websnark-bn128-pairings-v8-with-bignums.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## plot function for EVM vs Wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plotEVMvsWasm(df_evm_vs_wasm, colors=['tab:blue', 'tab:orange'], dodge=False):\n", + " f = plt.figure(figsize=(14,8))\n", + " #sns.set()\n", + " sns.reset_defaults() # use default colors, orange and blue\n", + " #sns.set(font_scale = 1.2)\n", + " splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=dodge,\n", + " #hue_order=['blake2b-8kb.huff.evm', 'blake2b-8kb.rust.wasm'],\n", + " #hue_order=['blake2b-8kb.rust.wasm', 'blake2b-8kb.huff.evm'],\n", + " #order=['blake2b-8kb.huff.evm', 'blake2b-8kb.rust.wasm'],\n", + " #order='exec_time',\n", + " #order=[\"\"]\n", + " palette=colors,\n", + " data=df_evm_vs_wasm)\n", + "\n", + "\n", + " # annotating bars: https://github.com/mwaskom/seaborn/issues/1582\n", + " for p in splot.patches:\n", + " splot.annotate(\n", + " #format(p.get_height(), '.4f'),\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + " splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + " plt.legend(fontsize='x-large', title_fontsize='40')\n", + " #f.suptitle(\"title\", fontsize=18, y=1.02)\n", + " return f\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EVM vs wasm shootout: bn128mul (Weierstrudel vs bn128mul.rust.wasm)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw.columns = df_evm_raw.columns.str.replace(' ','')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_raw[df_evm_raw['test_name'] == \"bn128_mul_weierstrudel-chfast2\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get wasm data for bn128_mul-chfast2\n", + "df_wasm_bn128mul = df_interp[df_interp['test_name'] == \"bn128_mul-chfast2\"].copy()\n", + "df_wasm_bn128mul.rename(columns={'test_name':'bench_name'},inplace=True)\n", + "df_wasm_bn128mul.drop(['elapsed_time'],axis=1,inplace=True)\n", + "df_wasm_bn128mul.drop(['total_time'],axis=1,inplace=True)\n", + "#df_wasm_mul256 = df_wasm_mul256[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "df_wasm_bn128mul.reset_index(inplace=True)\n", + "df_wasm_bn128mul.replace('bn128_mul-chfast2', 'bn128mul.rust.wasm', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get evm data\n", + "df_evm_bn128mul = df_evm_raw[df_evm_raw['test_name'] == \"bn128_mul_weierstrudel-chfast2\"].copy()\n", + "df_evm_bn128mul.rename(columns={'test_name':'bench_name', 'total_time':'exec_time'},inplace=True)\n", + "df_evm_bn128mul.drop(['gas_used'],axis=1,inplace=True)\n", + "df_evm_bn128mul.loc[:, 'parse_time'] = 0\n", + "df_evm_bn128mul = df_evm_bn128mul[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "\n", + "df_evm_bn128mul.replace('bn128_mul_weierstrudel-chfast2', 'bn128mul.weierstrudel.evm', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_vs_wasm_bn128mul = df_evm_bn128mul.append(df_wasm_bn128mul, sort=False).copy()\n", + "\n", + "\n", + "df_evm_vs_wasm_bn128mul.loc[:,'time'] = df_evm_vs_wasm_bn128mul['parse_time'].astype(float) + df_evm_vs_wasm_bn128mul['exec_time'].astype(float)\n", + "\n", + "df_evm_vs_wasm_bn128mul = df_evm_vs_wasm_bn128mul[df_evm_vs_wasm_bn128mul['engine'].isin(\n", + " ['parity-evm', 'geth-evm', 'evmone', 'cita-evm',\n", + " 'ssvm', 'wasmi', 'wabt', 'fizzy', 'wasm3']\n", + ")].copy()\n", + "\n", + "df_evm_vs_wasm_bn128mul = df_evm_vs_wasm_bn128mul.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_evm_vs_wasm_bn128mul[df_evm_vs_wasm_bn128mul['bench_name'] == '']\n", + "df_evm_vs_wasm_bn128mul.replace('bn128mul.weierstrudel.evm', 'EVM (weierstrudel.huff)', inplace=True)\n", + "df_evm_vs_wasm_bn128mul.replace('bn128mul.rust.wasm', 'Wasm (bn128mul.rs)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plotEVMvsWasm(df_evm_vs_wasm_bn128mul)\n", + "\n", + "#f.suptitle(\"EVM vs Wasm interpreters: bn128mul aka ECMUL \\n\\n bn128mul (Rust compiled to Wasm) vs Weierstrudel (Huff compiled to EVM)\", fontsize=18, y=1.02)\n", + "f.suptitle(\"EVM vs Wasm interpreters: ECMUL aka bn128mul \\n\\n Weierstrudel.huff (Huff compiled to EVM) vs bn128mul.rs (Rust compiled to Wasm)\", fontsize=18, y=1.02)\n", + "f.savefig('../images/evm-vs-wasm-bn128mul-huff-rust.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## mul256" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wasm_mul256 = df_interp[df_interp['test_name'] == \"mul256-gcolvin-drag-race\"].copy()\n", + "\n", + "\n", + "\n", + "df_wasm_mul256.rename(columns={'test_name':'bench_name'},inplace=True)\n", + "df_wasm_mul256.drop(['elapsed_time'],axis=1,inplace=True)\n", + "df_wasm_mul256.drop(['total_time'],axis=1,inplace=True)\n", + "#df_wasm_mul256 = df_wasm_mul256[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "df_wasm_mul256.reset_index(inplace=True)\n", + "df_wasm_mul256.replace('mul256-gcolvin-drag-race', 'mul256.rust.wasm', inplace=True)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_mul256 = df_evm_raw[df_evm_raw['test_name'] == \"mul256-gcolvin-drag-race\"].copy()\n", + "\n", + "df_evm_mul256.rename(columns={'test_name':'bench_name', 'total_time':'exec_time'},inplace=True)\n", + "df_evm_mul256.drop(['gas_used'],axis=1,inplace=True)\n", + "df_evm_mul256.loc[:, 'parse_time'] = 0\n", + "df_evm_mul256 = df_evm_mul256[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "\n", + "df_evm_mul256.replace('mul256-gcolvin-drag-race', 'mul256.sol.evm', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_vs_wasm_mul256 = df_evm_mul256.append(df_wasm_mul256, sort=False).copy()\n", + "df_evm_vs_wasm_mul256.loc[:,'time'] = df_evm_vs_wasm_mul256['parse_time'].astype(float) + df_evm_vs_wasm_mul256['exec_time'].astype(float)\n", + "\n", + "df_evm_vs_wasm_mul256 = df_evm_vs_wasm_mul256[df_evm_vs_wasm_mul256['engine'].isin(\n", + " ['parity-evm', 'geth-evm', 'evmone', 'cita-evm',\n", + " 'ssvm', 'wasmi', 'wabt', 'wamr-interp', 'fizzy', 'wasm3']\n", + ")].copy()\n", + "\n", + "df_evm_vs_wasm_mul256 = df_evm_vs_wasm_mul256.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_vs_wasm_mul256.replace('mul256.sol.evm', 'EVM (Solidity)', inplace=True)\n", + "df_evm_vs_wasm_mul256.replace('mul256.rust.wasm', 'Wasm (Rust)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plotEVMvsWasm(df_evm_vs_wasm_mul256)\n", + "\n", + "f.suptitle(\"EVM vs Wasm interpreters: mul256-gcolvin-drag-race (Solidity to EVM) and (Rust to Wasm)\", fontsize=18, y=0.95)\n", + "f.savefig('../images/evm-vs-wasm-mul256-sol-rust.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EVM vs wasm shootout: blake2b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_blake2b = df_evm_raw[df_evm_raw['test_name'].isin(['blake2b-8415-bytes', 'blake2b_huff-8415-bytes'])].copy()\n", + "\n", + "df_evm_blake2b.rename(columns={'test_name':'bench_name', 'total_time':'exec_time'},inplace=True)\n", + "df_evm_blake2b.drop(['gas_used'],axis=1,inplace=True)\n", + "df_evm_blake2b.loc[:, 'parse_time'] = 0\n", + "df_evm_blake2b = df_evm_blake2b[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "\n", + "df_evm_blake2b.replace('blake2b-8415-bytes', 'blake2b-8kb.sol.evm', inplace=True)\n", + "df_evm_blake2b.replace('blake2b_huff-8415-bytes', 'blake2b-8kb.huff.evm', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_wasm_blake2b = df_interp[df_interp['test_name'] == \"blake2b-8415-bytes\"].copy()\n", + "\n", + "df_wasm_blake2b.rename(columns={'test_name':'bench_name'},inplace=True)\n", + "df_wasm_blake2b.drop(['elapsed_time'],axis=1,inplace=True)\n", + "df_wasm_blake2b.drop(['total_time'],axis=1,inplace=True)\n", + "#df_wasm_blake2b = df_wasm_blake2b[['engine', 'bench_name', 'parse_time', 'exec_time']]\n", + "df_wasm_blake2b.reset_index(inplace=True)\n", + "df_wasm_blake2b.replace('blake2b-8415-bytes', 'blake2b-8kb.rust.wasm', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_vs_wasm_blake2b = df_evm_blake2b.append(df_wasm_blake2b, sort=False).copy()\n", + "\n", + "df_evm_vs_wasm_blake2b.loc[:,'time'] = df_evm_vs_wasm_blake2b['parse_time'].astype(float) + df_evm_vs_wasm_blake2b['exec_time'].astype(float)\n", + "\n", + "df_evm_vs_wasm_blake2b = df_evm_vs_wasm_blake2b[df_evm_vs_wasm_blake2b['engine'].isin(\n", + " ['parity-evm', 'geth-evm', 'evmone', 'cita-evm',\n", + " 'ssvm', 'wasmi', 'wabt', 'wamr-interp', 'fizzy', 'wasm3']\n", + ")].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm_vs_wasm_blake2b_sorted = df_evm_vs_wasm_blake2b.sort_values('time')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### blake2b.huff.evm vs blake2b.rust.wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_huff_vs_wasm = df_evm_vs_wasm_blake2b[df_evm_vs_wasm_blake2b['bench_name'].isin(\n", + " ['blake2b-8kb.huff.evm', 'blake2b-8kb.rust.wasm']\n", + ")].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_huff_vs_wasm = df_blake2b_huff_vs_wasm.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_huff_vs_wasm.replace('blake2b-8kb.huff.evm', 'EVM (Huff)', inplace=True)\n", + "df_blake2b_huff_vs_wasm.replace('blake2b-8kb.rust.wasm', 'Wasm (Rust)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plotEVMvsWasm(df_blake2b_huff_vs_wasm, colors=['tab:orange', 'tab:blue'])\n", + "\n", + "f.suptitle(\"EVM vs Wasm: BLAKE2b-8415-bytes (Huff to EVM) and (Rust to wasm)\", fontsize=18, y=0.95)\n", + "f.savefig('../images/evm-vs-wasm-blake2b-huff-rust.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### blake2b.sol.evm vs blake2b.huff.evm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_vs_huff = df_evm_vs_wasm_blake2b[df_evm_vs_wasm_blake2b['bench_name'].isin(\n", + " ['blake2b-8kb.huff.evm', 'blake2b-8kb.sol.evm']\n", + ")].copy()\n", + "\n", + "df_blake2b_sol_vs_huff = df_blake2b_sol_vs_huff.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_vs_huff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_vs_huff.replace('blake2b-8kb.huff.evm', 'EVM (Huff)', inplace=True)\n", + "df_blake2b_sol_vs_huff.replace('blake2b-8kb.sol.evm', 'EVM (Solidity)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plotEVMvsWasm(df_blake2b_sol_vs_huff, colors=['tab:blue', 'tab:olive'], dodge=True)\n", + "\n", + "f.suptitle(\"EVM implementations of BLAKE2b: Solidity vs Huff\", fontsize=18, y=0.95)\n", + "f.savefig('../images/evm-vs-wasm-blake2b-sol-huff.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## plot blake2b.sol.evm against blake2b.rust.wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_vs_rust = df_evm_vs_wasm_blake2b[df_evm_vs_wasm_blake2b['bench_name'].isin(\n", + " ['blake2b-8kb.sol.evm', 'blake2b-8kb.rust.wasm']\n", + ")].copy()\n", + "\n", + "df_blake2b_sol_vs_rust = df_blake2b_sol_vs_rust.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_vs_rust.replace('blake2b-8kb.rust.wasm', 'Wasm (Rust)', inplace=True)\n", + "df_blake2b_sol_vs_rust.replace('blake2b-8kb.sol.evm', 'EVM (Solidity)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plotEVMvsWasm(df_blake2b_sol_vs_rust, colors=['tab:orange', 'tab:olive'], dodge=False)\n", + "\n", + "f.suptitle(\"EVM vs Wasm interpreters: BLAKE2b-8415-bytes (Solidity-evm vs Rust-wasm)\", fontsize=18, y=0.95)\n", + "f.savefig('../images/evm-vs-wasm-blake2b-sol-rust.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## blake2b: solidity vs huff vs rust" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_huff_rust = df_evm_vs_wasm_blake2b[df_evm_vs_wasm_blake2b['bench_name'].isin(\n", + " ['blake2b-8kb.sol.evm', 'blake2b-8kb.huff.evm', 'blake2b-8kb.rust.wasm']\n", + ")].copy()\n", + "\n", + "df_blake2b_sol_huff_rust = df_blake2b_sol_huff_rust.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_blake2b_sol_huff_rust.replace('blake2b-8kb.rust.wasm', 'Wasm (Rust)', inplace=True)\n", + "df_blake2b_sol_huff_rust.replace('blake2b-8kb.sol.evm', 'EVM (Solidity)', inplace=True)\n", + "df_blake2b_sol_huff_rust.replace('blake2b-8kb.huff.evm', 'EVM (Huff)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "sns.reset_defaults() # use default colors, orange and blue\n", + "splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=True,\n", + " palette=['tab:orange', 'tab:blue', 'tab:olive'],\n", + " data=df_blake2b_sol_huff_rust)\n", + "\n", + "for patch in splot.patches:\n", + " current_width = patch.get_width()\n", + " diff = current_width*1.3 - current_width\n", + "\n", + " # orange - couldn't find a way to look up patch color names nor covert RGB to name\n", + " if patch.get_facecolor() == (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0):\n", + " diff = current_width*2 - current_width\n", + " patch.set_width(current_width*2)\n", + " patch.set_x(patch.get_x() + diff * .5)\n", + "\n", + " #blue\n", + " if patch.get_facecolor() == (0.19460784313725488, 0.45343137254901944, 0.632843137254902, 1.0):\n", + " patch.set_width(current_width*1.3)\n", + " patch.set_x(patch.get_x() - diff * 2.2)\n", + "\n", + " #olive\n", + " if patch.get_facecolor() == (0.6622549019607844, 0.6651960784313725, 0.2093137254901961, 1.0):\n", + " patch.set_width(current_width*1.3)\n", + " patch.set_x(patch.get_x() - diff * 1.5)\n", + "\n", + " #print(dir(patch))\n", + " #print(patch.get_facecolor())\n", + "\n", + "\n", + "for p in splot.patches:\n", + " splot.annotate(\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + "\n", + "plt.legend(fontsize='x-large', title_fontsize='40')\n", + "leg = splot.get_legend()\n", + "leg.set_bbox_to_anchor([0.3,0.9])\n", + "\n", + "splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + "\n", + "\n", + "\n", + "f.suptitle(\"EVM vs Wasm: BLAKE2b-8415-bytes \\n\\n Huff-evm vs Solidity-evm vs Rust-wasm\", fontsize=18, y=1.02)\n", + "f.savefig('../images/evm-vs-wasm-blake2b-huff-sol-rust.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BLS benchmarks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_data[df_scout_data['bench_name'].str.contains('bls')]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-no-bignums', 'wabt-fastmont-fasthost-superops']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-two-pairings',\n", + " 'bls12-eip1962-rust-native-two-pairings']\n", + "\n", + "\n", + "df_scout_bls12 = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-with-bignums']\n", + " ])\n", + "\n", + "df_scout_bls12 = df_scout_bls12.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_bls12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicks(df_scout_bls12,\n", + " suptitle=\"BLS12-381 two-point pairing check (Rust-eip1962 vs Wasm-wasmsnark)\",\n", + " suptitle_pos=1.02,\n", + " #subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " #subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-with-bignums\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-rust-vs-wasm-total-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_bls12,\n", + " suptitle=\"BLS12-381 two-point pairing check (Rust-eip1962 vs Wasm-wasmsnark)\",\n", + " suptitle_pos=1.07,\n", + " #subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " #subtitle_size='xx-large'\n", + " )\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-rust-vs-wasm-exec-startup-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## spectrum of bignum host funcs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['rust-native', 'v8-liftoff', 'v8-turbofan',\n", + " 'wabt-no-bignums', 'wabt-fastmont-fasthost-superops',\n", + " 'wabt-fastmont-fasthost-f1m_mul', 'wabt-fastmont-fasthost-f1m_mul-f1m_add',\n", + " 'wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub', 'wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul',\n", + " 'wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add', 'wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add-int_sub',\n", + " 'wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add-int_sub-int_div']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-two-pairings',\n", + " 'bls12-eip1962-rust-native-two-pairings']\n", + "\n", + "rename_engines= [\n", + " ['wabt-fastmont-fasthost-f1m_mul', 'wabt-bignums-MULMODMONT'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add', 'wabt-bignums-MULMODMONT-ADDMOD'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub', 'wabt-bignums-MULMODMONT-ADDMOD-SUBMOD'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul', 'wabt-bignums-*MOD-INTMUL'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add', 'wabt-bignums-*MOD-INTMUL-INTADD'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add-int_sub', 'wabt-bignums-*MOD-INTMUL-INTADD-INTSUB'],\n", + " ['wabt-fastmont-fasthost-f1m_mul-f1m_add-f1m_sub-int_mul-int_add-int_sub-int_div', 'wabt-bignums-*MOD-INTMUL-INTADD-INTSUB-INTDIV'],\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-bignums-all-hostfuncs-and-superops'],\n", + " ['wabt-bignums-slowhost-slowmont', 'wabt-with-bignums']\n", + "]\n", + "\n", + "\n", + "\n", + "df_scout_bls12_hostfunc_variations = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=rename_engines\n", + " )\n", + "\n", + "df_scout_bls12_hostfunc_variations = df_scout_bls12_hostfunc_variations.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestColoredTicksBLS(df_scout_bls12_hostfunc_variations,\n", + " suptitle=\"BLS12-381 two-point pairing check (Rust-eip1962 vs Wasm-wasmsnark)\",\n", + " suptitle_pos=1.02,\n", + " #subtitle=\"ecpairing-zkrollup-rust-wasm-bn128-two-pairings\\n\",\n", + " #subtitle_size='xx-large',\n", + " highlight_ticks=[\"wabt-with-bignums\"])\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-host-func-variations-total-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_scout_bls12_hostfunc_variations,\n", + " suptitle=\"BLS12-381 two-point pairing check (Rust-eip1962 vs Wasm-wasmsnark)\",\n", + " suptitle_pos=1.07,\n", + " #subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " #subtitle_size='xx-large'\n", + " )\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-host-func-variations-exec-startup-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BLS12 - wabt vs wabt-with-bignums vs native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.style.use('seaborn-white')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['rust-native', 'wabt-fastmont-fasthost-superops',\n", + " 'wabt-no-bignums']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-two-pairings',\n", + " 'bls12-wasmsnark-two-pairings-standalone',\n", + " 'bls12-eip1962-rust-native-two-pairings']\n", + "\n", + "# bls12-wasmsnark-two-pairings-standalone\n", + "# bls12-wasmsnark-synth-loop\n", + "\n", + "df_wabt_bls12 = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-with-bignums']\n", + " ])\n", + "\n", + "df_wabt_bls12 = df_wabt_bls12.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_wabt_bls12,\n", + " suptitle=\"BLS12-381 two-point pairing check\",\n", + " suptitle_pos=0.94,\n", + " subtitle=\"\",\n", + " #subtitle=\"wasmnark (on wabt and wabt-with-bigums) vs eip1962 (rust-native)\\n\",\n", + " #subtitle_size='xx-large'\n", + " )\n", + "\n", + "\n", + "fig_plt.grid(False)\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-wabt-vs-native-exec-startup-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.style.use('ggplot')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## fizzy vs wabt - BLS12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['rust-native', 'fizzy-with-bignums', 'wabt-fastmont-fasthost-superops',\n", + " 'wabt-no-bignums', 'fizzy']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-two-pairings',\n", + " 'bls12-wasmsnark-two-pairings-standalone',\n", + " 'bls12-eip1962-rust-native-two-pairings']\n", + "\n", + "# bls12-wasmsnark-two-pairings-standalone\n", + "# bls12-wasmsnark-synth-loop\n", + "\n", + "df_fizzy_bls12 = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-with-bignums']\n", + " ])\n", + "\n", + "df_fizzy_bls12 = df_fizzy_bls12.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_fizzy_bls12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_fizzy_bls12,\n", + " suptitle=\"BLS12-381 two-point pairing check (fizzy vs wabt, rust-eip1962 vs wasmsnark)\",\n", + " suptitle_pos=1.07,\n", + " #subtitle=\"ecpairing-zkrollup-websnark-bn128-two-pairings\\n\",\n", + " #subtitle_size='xx-large'\n", + " )\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-pairings-fizzy-vs-wabt-exec-startup-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BLS12 synth loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['fizzy-with-bignums', 'wabt-fastmont-fasthost-superops']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-synth-loop']\n", + "\n", + "# bls12-wasmsnark-two-pairings-standalone\n", + "# bls12-wasmsnark-synth-loop\n", + "\n", + "df_bls12_synth_loop = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-with-bignums']\n", + " ])\n", + "\n", + "df_bls12_synth_loop = df_bls12_synth_loop.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls12_synth_loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotScoutStackedTest(df_bls12_synth_loop,\n", + " suptitle=\"BLS12-381 synthetic loop (fizzy vs wabt - wasmsnark)\",\n", + " suptitle_pos=0.96,\n", + " subtitle=\"\",\n", + " #subtitle_size='xx-large'\n", + " )\n", + "\n", + "fig_plt.savefig('../images/bignums-bls12-synth-loop-fizzy-vs-wabt-exec-startup-time.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "include_engines_list = ['fizzy-with-bignums', 'wabt-fastmont-fasthost-superops']\n", + "\n", + "include_bench_names = ['bls12-wasmsnark-two-pairings-standalone', 'bls12-wasmsnark-synth-loop']\n", + "\n", + "df_bls12_synth_vs_pairing = \\\n", + " prepDfForRollupPlot(df_scout_data,\n", + " None,\n", + " include_bench_names,\n", + " only_include_engines=include_engines_list,\n", + " rename_engines=[\n", + " ['bls12-wasmsnark-two-pairings-standalone', 'bls12-two-pairings'],\n", + " ['bls12-wasmsnark-synth-loop', 'bls12-synth-loop'],\n", + " ['wabt-fastmont-fasthost-superops', 'wabt-with-bignums']\n", + " ])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestGroupedExexTime(df_bls12_synth_vs_pairing,\n", + " #title=\"BLS12-381 two pairings vs synthetic loop - wasmsnark (fizzy vs wabt)\",\n", + " title=\"Figure 2: BLS12-381 two pairings vs synthetic loop\",\n", + " test_title=\"\",\n", + " group_order=['bls12-synth-loop', 'bls12-two-pairings'],\n", + " sort_by=['bls12-synth-loop'],\n", + " colors=['tab:orange', 'tab:blue'])\n", + "\n", + "\n", + "fig_plt.savefig('../images/evm384-bls12-synth-loop-vs-pairings.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot evm384 bls synth loop results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384 = df_evm_raw[df_evm_raw['engine'] == 'evmone384']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_native = df_scout_data[df_scout_data['engine'] == 'rust-native']\n", + "\n", + "df_scout_rust_native_bls12 = df_scout_rust_native[df_scout_rust_native['bench_name'] == 'bls12-eip1962-rust-native-two-pairings']\n", + "\n", + "df_scout_rust_native_bls12 = df_scout_rust_native_bls12.groupby(['engine']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# proposed gas from EIP-2537, for a BLS12-381 pairing operation is 23000*k + 115000 where k is a number of pairs.\n", + "\n", + "eip2537_proposed_gas = 23000*2 + 115000\n", + "\n", + "df_scout_rust_native_bls12['gas'] = eip2537_proposed_gas\n", + "df_scout_rust_native_bls12['test_name'] = \"bls12-eip1962-rust-native-two-pairings\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_native_bls12" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_native_bls12_merge = df_scout_rust_native_bls12.copy()\n", + "df_scout_rust_native_bls12_merge.drop(columns=['parse_time'], inplace=True)\n", + "df_scout_rust_native_bls12_merge.reset_index(inplace=True)\n", + "df_scout_rust_native_bls12_merge.columns = ['engine', 'time', 'gas', 'test_name']\n", + "df_scout_rust_native_bls12_merge = df_scout_rust_native_bls12_merge[['engine','test_name','time','gas']]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_scout_rust_native_bls12_merge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_merge = df_evm384.copy()\n", + "df_evm384_merge.columns = ['engine', 'test_name', 'time', 'gas']\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_merge" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TODO: scale synthetic loop time/gas by a factor determined from the wasm runtimes (wasm-pairing / wasm-synth-loop)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native = pd.concat([df_evm384_merge, df_scout_rust_native_bls12_merge], sort=False)\n", + "df_evm384_native.reset_index(drop=True, inplace=True)\n", + "df_evm384_native.set_index('engine', inplace=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native['time'] = pd.to_numeric(df_evm384_native['time'])\n", + "df_evm384_native['gas'] = pd.to_numeric(df_evm384_native['gas'])\n", + "\n", + "df_evm384_native['Mgas/s'] = (df_evm384_native['gas'] / df_evm384_native['time']) * (1 / 10**6)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def byEngineExecTime(df_benchdata, engine):\n", + " #print(\"engine:\", engine)\n", + " #print(\"min_time:\", df_benchdata.loc[engine]['exec_time'].max())\n", + " return df_benchdata.loc[engine]['time'].max()\n", + "\n", + "\n", + "def plotTimeVsGas(df_benchdata, title=\"\"):\n", + " #sorted_engines = sorted(['evmone384', 'rust-native'], key=lambda x: byEngineExecTime(df_benchdata, x))\n", + " columns_to_plot = ['time']\n", + " df_plotdata = df_benchdata.copy()\n", + " df_plotdata = df_plotdata.set_index('test_name')\n", + " plt.figure()\n", + " f, subplot_axes = plt.subplots(1, 2, sharex=False, figsize=(10,3))\n", + "\n", + " time_axis = subplot_axes[0]\n", + " gas_axis = subplot_axes[1]\n", + " #time_axis.set_title(sorted_engines[i])\n", + "\n", + " df_plotdata[['time']].plot.barh(ax=time_axis, color=['steelblue'])\n", + " # set blank to hide \"test_name\"\n", + " time_axis.set(ylabel='')\n", + " time_axis.legend(loc='upper right', bbox_to_anchor=(1.0, 1.2))\n", + "\n", + " # label lower axis on bottom plot\n", + " time_axis.set(ylabel='', xlabel='seconds')\n", + "\n", + " #for i, engine in enumerate(sorted_engines):\n", + " labelBarWidths(time_axis)\n", + "\n", + " df_plotdata[['gas']].plot.barh(ax=gas_axis, color=['olive'])\n", + "\n", + " gas_axis.legend(loc='upper right', bbox_to_anchor=(1.0, 1.2))\n", + "\n", + " mgas0 = df_plotdata.iloc[0]['Mgas/s']\n", + " mgas1 = df_plotdata.iloc[1]['Mgas/s']\n", + " mgas2 = df_plotdata.iloc[2]['Mgas/s']\n", + " \n", + " mgas_txt = \"{} Mgas/s\".format(round(mgas0,1))\n", + " gas_axis.text(1.05, 0.2, mgas_txt, fontsize=10, transform=gas_axis.transAxes)\n", + "\n", + " mgas_txt = \"{} Mgas/s\".format(round(mgas1,1))\n", + " gas_axis.text(1.05, 0.5, mgas_txt, fontsize=10, transform=gas_axis.transAxes)\n", + "\n", + " mgas_txt = \"{} Mgas/s\".format(round(mgas2,1))\n", + " gas_axis.text(1.05, 0.8, mgas_txt, fontsize=10, transform=gas_axis.transAxes)\n", + " \n", + " \n", + " # label lower axis on bottom plot\n", + " gas_axis.set(ylabel='', xlabel='gas')\n", + " gas_axis.label_outer()\n", + "\n", + " \n", + " f.suptitle(title, fontsize=16, y=1.22)\n", + " f.subplots_adjust(hspace=0.5)\n", + " return f\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotTimeVsGas(df_evm384_native, \"bls12-381 pairings: rust-native precompile vs evm384 synthetic loop\")\n", + "\n", + "fig_plt.savefig('../images/evm384-bls12-synth-loop-vs-native-rust.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## bls12 pairings and synth loop - wasm, evm384, rust-native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls_wasm_merge = df_bls12_synth_vs_pairing.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls_wasm_merge.drop(columns=['parse_time', 'exec_time'], inplace=True)\n", + "\n", + "df_bls_wasm_merge['engine_bench_name'] = df_bls_wasm_merge['engine'] + \"--\" + df_bls_wasm_merge['bench_name']\n", + "df_bls_wasm_merge.drop(columns=['engine', 'bench_name'], inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls_wasm_merge = df_bls_wasm_merge.groupby(['engine_bench_name']).mean()\n", + "df_bls_wasm_merge.reset_index(inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls_wasm_merge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge = df_evm384_native.copy()\n", + "df_evm384_native_merge.drop(columns=['gas', 'Mgas/s'], inplace=True)\n", + "df_evm384_native_merge.reset_index(inplace=True)\n", + "df_evm384_native_merge.columns = ['engine', 'bench_name', 'time']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge['engine_bench_name'] = df_evm384_native_merge['engine'] + \"--\" + df_evm384_native_merge['bench_name']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge.drop(columns=['engine', 'bench_name'], inplace=True)\n", + "df_evm384_native_merge = df_evm384_native_merge[['engine_bench_name', 'time']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native = pd.concat([df_evm384_native_merge, df_bls_wasm_merge], sort=False)\n", + "df_evm384_wasm_native.reset_index(drop=True, inplace=True)\n", + "df_evm384_wasm_native.set_index('engine_bench_name', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestUsingTimeCol(df_evm384_wasm_native,\n", + " suptitle=\"BLS12-381 pairings and synthetic loop - rust-native, wasm, evm384\",\n", + " suptitle_pos=1.02)\n", + "\n", + "fig_plt.savefig('../images/evm384-bls12-pairings-synth-loop-rust-wasm.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_versions_and_native = df_evm384_wasm_native.copy()\n", + "df_evm384_versions_and_native.reset_index(inplace=True)\n", + "\n", + "df_evm384_versions_and_native = df_evm384_versions_and_native[~df_evm384_versions_and_native['engine_bench_name'].isin([\n", + " 'fizzy-with-bignums--bls12-synth-loop',\n", + " 'fizzy-with-bignums--bls12-two-pairings',\n", + " 'wabt-with-bignums--bls12-synth-loop',\n", + " 'wabt-with-bignums--bls12-two-pairings'\n", + "])]\n", + "\n", + "df_evm384_versions_and_native.loc[df_evm384_versions_and_native['engine_bench_name'].str.contains(pat='evmone'), 'time'] *= 1.25\n", + "\n", + "#df_evm384_versions_and_native.drop(df_evm384_versions_and_native['engine_bench_name'].isin([\n", + "# 'fizzy-with-bignums--bls12-synth-loop',\n", + "# 'fizzy-with-bignums--bls12-two-pairings',\n", + "# 'wabt-with-bignums--bls12-synth-loop',\n", + "# 'wabt-with-bignums--bls12-two-pairings'\n", + "#]))\n", + "\n", + "df_evm384_versions_and_native.set_index('engine_bench_name', inplace=True)\n", + "\n", + "fig_plt = plotOneTestUsingTimeCol(df_evm384_versions_and_native,\n", + " suptitle=\"bls12-pairing: evm384 (estimated via adjustment factor) vs rust native\",\n", + " suptitle_pos=1.02)\n", + "\n", + "fig_plt.savefig('../images/evm384-versions-vs-rust-native.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_synth_loop = df_evm384_wasm_native.copy()\n", + "df_evm384_wasm_synth_loop.reset_index(inplace=True)\n", + "\n", + "df_evm384_wasm_synth_loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_synth_loop = df_evm384_wasm_synth_loop[df_evm384_wasm_synth_loop['engine_bench_name'].str.contains(\"synth\")]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_synth_loop.set_index('engine_bench_name', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig_plt = plotOneTestUsingTimeCol(df_evm384_wasm_synth_loop,\n", + " suptitle=\"BLS12-381 synthetic loop - wasm and evm384\",\n", + " suptitle_pos=1.02)\n", + "\n", + "fig_plt.savefig('../images/evm384-bls12-synth-loop-wasm.png', bbox_inches='tight')\n", + "fig_plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## evm384 concluding tldr chart" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_bls_wasm_merge_tldr = df_bls12_synth_vs_pairing.copy()\n", + "df_bls_wasm_merge_tldr.drop(columns=['parse_time', 'exec_time'], inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_native_merge_tldr = df_evm384_native.copy()\n", + "df_evm384_native_merge_tldr.drop(columns=['gas', 'Mgas/s'], inplace=True)\n", + "df_evm384_native_merge_tldr.reset_index(inplace=True)\n", + "df_evm384_native_merge_tldr.columns = ['engine', 'bench_name', 'time']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr = pd.concat([df_evm384_native_merge_tldr, df_bls_wasm_merge_tldr], sort=False)\n", + "df_evm384_wasm_native_tldr.reset_index(drop=True, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr = df_evm384_wasm_native_tldr.drop(df_evm384_wasm_native_tldr[df_evm384_wasm_native_tldr['engine'] == 'fizzy-with-bignums'].index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr = df_evm384_wasm_native_tldr.sort_values('time')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr.replace('bls12-eip1962-rust-native-two-pairings', 'Native two-pairings (Rust)', inplace=True)\n", + "df_evm384_wasm_native_tldr.replace('evm384-synth-loop-v3', 'EVM384 synth-loop-v3 (Yul)', inplace=True)\n", + "df_evm384_wasm_native_tldr.replace('evm384-synth-loop-v2', 'EVM384 synth-loop-v2 (Yul)', inplace=True)\n", + "df_evm384_wasm_native_tldr.replace('evm384-synth-loop-v1', 'EVM384 synth-loop-v1 (Yul)', inplace=True)\n", + "df_evm384_wasm_native_tldr.replace('bls12-synth-loop', 'Wasm synth-loop (wasmsnark)', inplace=True)\n", + "df_evm384_wasm_native_tldr.replace('bls12-two-pairings', 'Wasm two-pairings (wasmsnark)', inplace=True)\n", + "#df_evm384_wasm_native_tldr.replace('blake2b-8kb.huff.evm', 'EVM (Huff)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_synth_pairing_yul = df_evm384_wasm_native_tldr.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# drop rust-native - this chart focuses on synth loop vs pairing (and evmone384 synth loop)\n", + "df_evm384_wasm_synth_pairing_yul = df_evm384_wasm_synth_pairing_yul.drop(df_evm384_wasm_synth_pairing_yul[df_evm384_wasm_synth_pairing_yul['engine'] == 'rust-native'].index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "sns.reset_defaults() # use default colors, orange and blue\n", + "splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=True, ci=None,\n", + " palette=['tab:blue', 'tab:olive', 'tab:purple', 'tab:brown', 'tab:cyan'],\n", + " data=df_evm384_wasm_synth_pairing_yul)\n", + "\n", + "\n", + "\n", + "# this customizes the widths of each bar-group. Because orange bars (wasm) stand alone,\n", + "# versus blue and olive that are grouped together (evm-huff, evm-sol), but the default\n", + "# widths are all the same, the orange bars are too skinny.\n", + "# This makes the orange bars fatter (`current_width*2`) than the blue and olive bars (`current_width*1.3`)\n", + "\n", + "# It also adjusts the x position of the bars so that they're centered (by default they were positioned\n", + "# as if all three groups would appear in every column)\n", + "\n", + "orange_bars_rgba = (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "blue_bars_rgba = (0.19460784313725488, 0.45343137254901944, 0.632843137254902, 1.0)\n", + "olive_bars_rgba = (0.6622549019607844, 0.6651960784313725, 0.2093137254901961, 1.0)\n", + "purple_bars_rgba = (0.5784313725490194, 0.446078431372549, 0.6990196078431372, 1.0)\n", + "brown_bars_rgba = (0.5171568627450981, 0.3583333333333334, 0.3259803921568628, 1.0)\n", + "cyan_bars_rgba = (0.180392156862745, 0.6715686274509803, 0.7215686274509805, 1.0)\n", + "\n", + "\n", + "for patch in splot.patches:\n", + " current_width = patch.get_width()\n", + " # factor 1.3 to make bars 30% wider?\n", + " diff = current_width*1.3 - current_width\n", + "\n", + " # couldn't find a way to look up patch color names nor covert RGB to name,\n", + " # so we look up the bar groups using these HSLA(?) values like (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "\n", + " #blue\n", + " if patch.get_facecolor() == blue_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 3.4)\n", + "\n", + " #olive\n", + " if patch.get_facecolor() == olive_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 3.4)\n", + "\n", + " # purple\n", + " if patch.get_facecolor() == purple_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + "\n", + " # brown\n", + " if patch.get_facecolor() == brown_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + " \n", + " # cyan\n", + " if patch.get_facecolor() == cyan_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + " \n", + " \n", + " #print(dir(patch))\n", + " #print(patch)\n", + " #print(patch.get_facecolor())\n", + "\n", + "\n", + "# annotate the bars with the ms values\n", + "for p in splot.patches:\n", + " splot.annotate(\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + "\n", + "# adjust legend placement\n", + "plt.legend(fontsize='x-large', title_fontsize='40', loc='upper left')\n", + "#leg = splot.get_legend()\n", + "#leg.set_bbox_to_anchor([0.212,0.95])\n", + "\n", + "splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + "splot.set(xlabel='')\n", + "\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "f.savefig('../images/evm384-wasm-native-pairings-and-synth-loop.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## just wasm - synth loop vs pairing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_wasm = df_evm384_wasm_native_tldr.copy()\n", + "# drop rust-native - this chart focuses on synth loop vs pairing (and evmone384 synth loop)\n", + "df_evm384_synth_wasm = df_evm384_synth_wasm.drop(df_evm384_synth_wasm[df_evm384_synth_wasm['engine'] == 'rust-native'].index)\n", + "\n", + "df_evm384_synth_wasm = df_evm384_synth_wasm.drop(df_evm384_synth_wasm[df_evm384_synth_wasm['engine'] == 'evmone384'].index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_evm384_synth_wasm.loc[(df_evm384_synth_wasm['bench_name'] == 'Wasm two-pairings (wasmsnark)'), 'engine'] = \"wasm-two-pairings\"\n", + "#df_evm384_synth_wasm.loc[(df_evm384_synth_wasm['bench_name'] == 'Wasm synth-loop (wasmsnark)'), 'engine'] = \"wasm-synth-loop\"\n", + "\n", + "df_evm384_synth_wasm.loc[(df_evm384_synth_wasm['bench_name'] == 'Wasm two-pairings (wasmsnark)'), 'engine'] = \"wabt-with-bignums (two-pairings)\"\n", + "df_evm384_synth_wasm.loc[(df_evm384_synth_wasm['bench_name'] == 'Wasm synth-loop (wasmsnark)'), 'engine'] = \"wabt-with-bignums (synth-loop)\"\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_wasm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "sns.reset_defaults() # use default colors, orange and blue\n", + "splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=True, ci=None,\n", + " palette=['tab:blue', 'tab:olive', 'tab:purple', 'tab:brown', 'tab:cyan'],\n", + " data=df_evm384_synth_wasm)\n", + "\n", + "\n", + "\n", + "# this customizes the widths of each bar-group. Because orange bars (wasm) stand alone,\n", + "# versus blue and olive that are grouped together (evm-huff, evm-sol), but the default\n", + "# widths are all the same, the orange bars are too skinny.\n", + "# This makes the orange bars fatter (`current_width*2`) than the blue and olive bars (`current_width*1.3`)\n", + "\n", + "# It also adjusts the x position of the bars so that they're centered (by default they were positioned\n", + "# as if all three groups would appear in every column)\n", + "\n", + "orange_bars_rgba = (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "blue_bars_rgba = (0.19460784313725488, 0.45343137254901944, 0.632843137254902, 1.0)\n", + "olive_bars_rgba = (0.6622549019607844, 0.6651960784313725, 0.2093137254901961, 1.0)\n", + "purple_bars_rgba = (0.5784313725490194, 0.446078431372549, 0.6990196078431372, 1.0)\n", + "brown_bars_rgba = (0.5171568627450981, 0.3583333333333334, 0.3259803921568628, 1.0)\n", + "cyan_bars_rgba = (0.180392156862745, 0.6715686274509803, 0.7215686274509805, 1.0)\n", + "\n", + "\n", + "for patch in splot.patches:\n", + " current_width = patch.get_width()\n", + " diff = current_width*1.3 - current_width\n", + "\n", + " # couldn't find a way to look up patch color names nor covert RGB to name,\n", + " # so we look up the bar groups using these HSLA(?) values like (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "\n", + " #blue\n", + " if patch.get_facecolor() == blue_bars_rgba:\n", + " patch.set_width(current_width*0.7)\n", + " patch.set_x(patch.get_x() + diff * 2.1)\n", + "\n", + " #olive\n", + " if patch.get_facecolor() == olive_bars_rgba:\n", + " patch.set_width(current_width*0.7)\n", + " patch.set_x(patch.get_x() - diff * 1.1)\n", + "\n", + " # purple\n", + " if patch.get_facecolor() == purple_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + "\n", + " # brown\n", + " if patch.get_facecolor() == brown_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + " \n", + " # cyan\n", + " if patch.get_facecolor() == cyan_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + " \n", + " \n", + " #print(dir(patch))\n", + " #print(patch)\n", + " #print(patch.get_facecolor())\n", + "\n", + "\n", + "# annotate the bars with the ms values\n", + "for p in splot.patches:\n", + " splot.annotate(\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + "\n", + "# adjust legend placement\n", + "plt.legend(fontsize='x-large', title_fontsize='40', loc='upper left')\n", + "#leg = splot.get_legend()\n", + "#leg.set_bbox_to_anchor([0.212,0.95])\n", + "\n", + "splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + "splot.set(xlabel='')\n", + "\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "f.savefig('../images/evm384-wasm-pairings-vs-synth-loop.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## just evmone384 - v1 v2 v3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3 = df_evm384_wasm_native_tldr.copy()\n", + "# drop rust-native - this chart focuses on synth loop vs pairing (and evmone384 synth loop)\n", + "df_evm384_synth_v1v2v3 = df_evm384_synth_v1v2v3.drop(df_evm384_synth_v1v2v3[df_evm384_synth_v1v2v3['engine'] == 'rust-native'].index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3 = df_evm384_synth_v1v2v3.drop(df_evm384_synth_v1v2v3[df_evm384_synth_v1v2v3['engine'] == 'wabt-with-bignums'].index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#df_evm384_synth_v1v2v3.replace('evm384-synth-loop-v3', 'EVM384 synth-loop-v3 (Yul)', inplace=True)\n", + "#df_evm384_synth_v1v2v3.replace('evm384-synth-loop-v2', 'EVM384 synth-loop-v2 (Yul)', inplace=True)\n", + "#df_evm384_synth_v1v2v3.replace('evm384-synth-loop-v1', 'EVM384 synth-loop-v1 (Yul)', inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3.loc[(df_evm384_synth_v1v2v3['bench_name'] == 'EVM384 synth-loop-v3 (Yul)'), 'engine'] = \"evmone-evm384-v3\"\n", + "df_evm384_synth_v1v2v3.loc[(df_evm384_synth_v1v2v3['bench_name'] == 'EVM384 synth-loop-v2 (Yul)'), 'engine'] = \"evmone-evm384-v2\"\n", + "df_evm384_synth_v1v2v3.loc[(df_evm384_synth_v1v2v3['bench_name'] == 'EVM384 synth-loop-v1 (Yul)'), 'engine'] = \"evmone-evm384-v1\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_synth_v1v2v3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "sns.reset_defaults() # use default colors, orange and blue\n", + "splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=True, ci=None,\n", + " palette=['tab:purple', 'tab:brown', 'tab:cyan'],\n", + " data=df_evm384_synth_v1v2v3)\n", + "\n", + "\n", + "\n", + "# this customizes the widths of each bar-group. Because orange bars (wasm) stand alone,\n", + "# versus blue and olive that are grouped together (evm-huff, evm-sol), but the default\n", + "# widths are all the same, the orange bars are too skinny.\n", + "# This makes the orange bars fatter (`current_width*2`) than the blue and olive bars (`current_width*1.3`)\n", + "\n", + "# It also adjusts the x position of the bars so that they're centered (by default they were positioned\n", + "# as if all three groups would appear in every column)\n", + "\n", + "orange_bars_rgba = (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "blue_bars_rgba = (0.19460784313725488, 0.45343137254901944, 0.632843137254902, 1.0)\n", + "olive_bars_rgba = (0.6622549019607844, 0.6651960784313725, 0.2093137254901961, 1.0)\n", + "purple_bars_rgba = (0.5784313725490194, 0.446078431372549, 0.6990196078431372, 1.0)\n", + "brown_bars_rgba = (0.5171568627450981, 0.3583333333333334, 0.3259803921568628, 1.0)\n", + "cyan_bars_rgba = (0.180392156862745, 0.6715686274509803, 0.7215686274509805, 1.0)\n", + "\n", + "\n", + "for patch in splot.patches:\n", + " current_width = patch.get_width()\n", + " # factor 1.3 to make bars 30% wider?\n", + " diff = current_width*1.3 - current_width\n", + "\n", + " # couldn't find a way to look up patch color names nor covert RGB to name,\n", + " # so we look up the bar groups using these HSLA(?) values like (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "\n", + " #blue\n", + " if patch.get_facecolor() == purple_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 3.4)\n", + "\n", + " #olive\n", + " if patch.get_facecolor() == brown_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 0.0)\n", + "\n", + " # purple\n", + " if patch.get_facecolor() == cyan_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.4)\n", + " \n", + " \n", + " #print(dir(patch))\n", + " #print(patch)\n", + " #print(patch.get_facecolor())\n", + "\n", + "\n", + "# annotate the bars with the ms values\n", + "for p in splot.patches:\n", + " splot.annotate(\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + "\n", + "# adjust legend placement\n", + "plt.legend(fontsize='x-large', title_fontsize='40', loc='upper left')\n", + "#leg = splot.get_legend()\n", + "#leg.set_bbox_to_anchor([0.212,0.95])\n", + "\n", + "splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + "splot.set(xlabel='')\n", + "\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Synthetic loop (wasm, evm384)\", fontsize=18, y=0.96)\n", + "f.savefig('../images/evm384-synth-loop-v1-v2-v3.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot the evm384 adjusted time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr_adjusted = df_evm384_wasm_native_tldr.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr_adjusted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wasm_pairing_time = df_evm384_wasm_native_tldr_adjusted.loc[(df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'Wasm two-pairings (wasmsnark)'), 'time']\n", + "wasm_synth_time = df_evm384_wasm_native_tldr_adjusted.loc[(df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'Wasm synth-loop (wasmsnark)'), 'time']\n", + "\n", + "adjustment_factor = wasm_pairing_time.mean() / wasm_synth_time.mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "adjustment_factor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr_adjusted = df_evm384_wasm_native_tldr_adjusted.drop(df_evm384_wasm_native_tldr_adjusted[df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'Wasm synth-loop (wasmsnark)'].index)\n", + "df_evm384_wasm_native_tldr_adjusted = df_evm384_wasm_native_tldr_adjusted.drop(df_evm384_wasm_native_tldr_adjusted[df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'EVM384 synth-loop-v1 (Yul)'].index)\n", + "df_evm384_wasm_native_tldr_adjusted = df_evm384_wasm_native_tldr_adjusted.drop(df_evm384_wasm_native_tldr_adjusted[df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'EVM384 synth-loop-v2 (Yul)'].index)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr_adjusted.loc[(df_evm384_wasm_native_tldr_adjusted['engine'] == 'evmone384'), 'time'] *= adjustment_factor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_evm384_wasm_native_tldr_adjusted.loc[(df_evm384_wasm_native_tldr_adjusted['bench_name'] == 'EVM384 synth-loop-v3 (Yul)'), 'engine'] = \"evmone-evm384-v3\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "evm384_legend_label = \"EVM384 estimate (after synthetic adjustment factor of {:0.2f}x)\".format(adjustment_factor)\n", + "\n", + "df_evm384_wasm_native_tldr_adjusted.replace('EVM384 synth-loop-v3 (Yul)', evm384_legend_label, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = plt.figure(figsize=(14,8))\n", + "sns.reset_defaults() # use default colors, orange and blue\n", + "splot = sns.barplot(x='engine', y='time', hue='bench_name', dodge=True, ci=None,\n", + " palette=['tab:orange', 'tab:olive', 'tab:pink'],\n", + " data=df_evm384_wasm_native_tldr_adjusted)\n", + "\n", + "\n", + "\n", + "# this customizes the widths of each bar-group. Because orange bars (wasm) stand alone,\n", + "# versus blue and olive that are grouped together (evm-huff, evm-sol), but the default\n", + "# widths are all the same, the orange bars are too skinny.\n", + "# This makes the orange bars fatter (`current_width*2`) than the blue and olive bars (`current_width*1.3`)\n", + "\n", + "# It also adjusts the x position of the bars so that they're centered (by default they were positioned\n", + "# as if all three groups would appear in every column)\n", + "\n", + "orange_bars_rgba = (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + "blue_bars_rgba = (0.19460784313725488, 0.45343137254901944, 0.632843137254902, 1.0)\n", + "olive_bars_rgba = (0.6622549019607844, 0.6651960784313725, 0.2093137254901961, 1.0)\n", + "purple_bars_rgba = (0.5784313725490194, 0.446078431372549, 0.6990196078431372, 1.0)\n", + "brown_bars_rgba = (0.5171568627450981, 0.3583333333333334, 0.3259803921568628, 1.0)\n", + "pink_bars_rgba = (0.8372549019607844, 0.5196078431372548, 0.7401960784313724, 1.0)\n", + "\n", + "\n", + "for patch in splot.patches:\n", + " current_width = patch.get_width()\n", + " # factor 1.3 to make bars 30% wider?\n", + " diff = current_width*1.3 - current_width\n", + "\n", + " # orange - couldn't find a way to look up patch color names nor covert RGB to name,\n", + " # so we look up the bar groups using these HSLA(?) values like (0.8818627450980391, 0.5053921568627451, 0.17303921568627467, 1.0)\n", + " if patch.get_facecolor() == orange_bars_rgba:\n", + " diff = current_width*2 - current_width\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 1.0)\n", + "\n", + " #olive\n", + " if patch.get_facecolor() == olive_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() + diff * 0.1)\n", + "\n", + " # pink\n", + " if patch.get_facecolor() == pink_bars_rgba:\n", + " patch.set_width(current_width*1.0)\n", + " patch.set_x(patch.get_x() - diff * 3.2)\n", + " \n", + "\n", + " #print(dir(patch))\n", + " #print(patch)\n", + " #print(patch.get_facecolor())\n", + "\n", + "\n", + "# annotate the bars with the ms values\n", + "for p in splot.patches:\n", + " splot.annotate(\n", + " toMs(p.get_height()),\n", + " (p.get_x() + p.get_width() / 2., p.get_height()),\n", + " ha = 'center', va = 'center',\n", + " xytext = (0, 10),\n", + " textcoords = 'offset points')\n", + "\n", + "\n", + "# adjust legend placement\n", + "plt.legend(fontsize='x-large', title_fontsize='40', loc='upper left')\n", + "#leg = splot.get_legend()\n", + "#leg.set_bbox_to_anchor([0.212,0.95])\n", + "\n", + "splot.set_xticklabels(splot.get_xticklabels(), size = 14)\n", + "splot.set(xlabel='')\n", + "\n", + "#f.suptitle(\"Two-pairings (Rust-native, wasm) vs Estimated-two-pairings (evm384 adjusted-synth-loop)\", fontsize=18, y=0.96)\n", + "#f.suptitle(\"BLS12-381: Estimated runtime for two pairings on EVM384\", fontsize=18, y=0.96)\n", + "f.savefig('../images/evm384-wasm-native-pairings-vs-adjusted-synth-loop.png', bbox_inches='tight')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# test to make sure that scout wasm3 data is present\n", + "test_scout_wasm3_plot = plotOneTestColoredTicks(df_scout_data[df_scout_data['engine'].isin(['wasm3-with-bignums'])],)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# test to make sure that scout_wamr.c data is present\n", + "test_scout_wasm3_plot = plotOneTestColoredTicks(df_scout_data[df_scout_data['engine'].isin(['wasm3-with-bignums'])],)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_evm384_yul_vs_huff(df):\n", + " yul_times = list(df[df['engine_bench_name'].str.contains(pat='yul')]['time']) + [0.0, 0.0, 0.0]\n", + "\n", + " x = list(df[df['engine_bench_name'].str.contains(pat='huff')]['time'])\n", + " huff_times = [0.0, 0.0, 0.0, x[1], 0.0, x[0], x[2], 0.0]\n", + "\n", + " yul_times = list(map(lambda x: x * 1000, yul_times))\n", + " huff_times = list(map(lambda x: x * 1000, huff_times))\n", + " wabt_time = [0.0 for i in range(7)] + list(df[df['engine_bench_name'] == 'wabt-with-bignums--bls12-synth-loop']['time'])\n", + " wabt_time = list(map(lambda x: x * 1000, wabt_time))\n", + " import pdb; pdb.set_trace()\n", + "\n", + " fig, ax = plt.subplots()\n", + "\n", + " ind = np.arange(8) # the x locations for the groups\n", + " width = 0.35 # the width of the bars\n", + " yul_bars = ax.bar(ind, yul_times, width, bottom=0)\n", + "\n", + " huff_bars = ax.bar(ind+width, huff_times, width, bottom=0)\n", + "\n", + " wabt_bar = ax.bar(ind + width / 2, wabt_time, width, bottom=0)\n", + "\n", + " ax.set_title('EVM384 implementations: f6m_mul synth loop benchmark')\n", + "\n", + " # TODO convert seconds to milliseconds\n", + "\n", + " # ax.set_title('Scores by group and gender')\n", + " ax.set_xticks(ind + width / 2)\n", + " ax.set_xticklabels(('v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'wasm'))\n", + " ax.set_ylabel('Milliseconds')\n", + "\n", + " ax.legend((yul_bars[0], huff_bars[0], wabt_bar[0]), ('Yul', 'Huff', 'Wabt'))\n", + " # ax.yaxis.set_units(inch)\n", + " ax.autoscale_view()\n", + "\n", + " plt.savefig('../images/evm384-all-versions.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plot_evm384_yul_vs_huff(df_evm384_versions_and_native)\n", + "\n", + "df_evm384_versions_summarized = df_evm384_versions_and_native.copy()\n", + "df_evm384_versions_summarized = df_evm384_versions_summarized[df_evm384_versions_summarized['engine_bench_name'].isin([\n", + "'evmone384--evm384-synth-loop-v7-huff',\n", + "'evmone384--evm384-synth-loop-v3-yul',\n", + "'rust-native--bls12-eip1962-rust-native-two-pairings',\n", + "'wabt-with-bignums--bls12-two-pairings'\n", + "])]\n", + "\n", + "df_evm384_versions_summarized.loc[df_evm384_versions_and_native['engine_bench_name'].str.contains(pat='evmone'), 'time'] *= 1.25\n", + "\n", + "df_evm384_versions_summarized.set_index('engine_bench_name', inplace=True)\n", + "\n", + "fig_plt = plotOneTestUsingTimeCol(df_evm384_versions_summarized,\n", + " suptitle=\"bls12-pairing: evm384 (estimated) vs native and interpreted wasm\",\n", + " suptitle_pos=1.02)\n", + "\n", + "fig_plt.savefig('../images/evm384-vs-native-wasm.png', bbox_inches='tight')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}