E2E o1VM testing with cached artifacts.

.github/workflows/ci.yml

@@ -77,7 +77,6 @@ jobs:
         # It might be related to boxroot dependency, and we would need to bump
         # up the ocaml-rs dependency
         ocaml_version: ["4.14"]
-        os: ["ubuntu-latest"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -89,7 +88,7 @@ jobs:
         with:
           rust_toolchain_version: ${{ matrix.rust_toolchain_version }}
 
-      - name: Rust Cache
+      - name: Apply the Rust smart cacheing
         uses: Swatinem/rust-cache@v2
 
       - name: Use shared OCaml setting up steps

.github/workflows/o1vm-ci.yml

@@ -0,0 +1,120 @@
+name: o1vm CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      [
+        "o1vm/**",
+        "folding/**",
+        "groupmap/**",
+        "kimchi/**",
+        "msm/**",
+        "curves/**",
+        "poseidon/**",
+        "poly-commitment/",
+        "internal-tracing/**",
+        "srs/**",
+        "utils/**",
+      ]
+  push:
+    branches:
+      - master
+    paths:
+      [
+        "o1vm/**",
+        "folding/**",
+        "groupmap/**",
+        "kimchi/**",
+        "msm/**",
+        "curves/**",
+        "poseidon/**",
+        "poly-commitment/",
+        "internal-tracing/**",
+        "srs/**",
+        "utils/**",
+      ]
+
+env:
+  # https://doc.rust-lang.org/cargo/reference/profiles.html#release
+  # Disable for the time being since it fails with the "attempt to multiply with overflow" error.
+  # Known issue yeat to be fixed.
+  # RUSTFLAGS: -Coverflow-checks=y -Cdebug-assertions=y
+  # https://doc.rust-lang.org/cargo/reference/profiles.html#incremental
+  CARGO_INCREMENTAL: 1
+  # https://nexte.st/book/pre-built-binaries.html#using-nextest-in-github-actions
+  CARGO_TERM_COLOR: always
+  # 30 MB of stack for Keccak tests
+  RUST_MIN_STACK: 31457280
+
+jobs:
+  run_o1vm_with_cached_data:
+    name: Run o1vm with cached data
+    # We run only one of the matrix options on the toffee `hetzner-1` self-hosted GitHub runner.
+    # Only in this configuration we enable tests with the code coverage data gathering.
+    runs-on: ["ubuntu-latest"]
+    strategy:
+      matrix:
+        rust_toolchain_version: ["1.74"]
+        # FIXME: currently not available for 5.0.0.
+        # It might be related to boxroot dependency, and we would need to bump
+        # up the ocaml-rs dependency
+        ocaml_version: ["4.14"]
+    services:
+      o1vm-e2e-testing-cache:
+        image: o1labs/proof-systems:o1vm-e2e-testing-cache
+        volumes:
+          - /tmp:/tmp/cache
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Use shared Rust toolchain setting up steps
+        uses: ./.github/actions/toolchain-shared
+        with:
+          rust_toolchain_version: ${{ matrix.rust_toolchain_version }}
+
+      - name: Apply the Rust smart cacheing
+        uses: Swatinem/rust-cache@v2
+
+      - name: Use shared OCaml setting up steps
+        uses: ./.github/actions/ocaml-shared
+        with:
+          ocaml_version: ${{ matrix.ocaml_version }}
+
+      - name: Install the Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          check-latest: true
+
+      - name: Install the Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.21.0"
+
+      - name: Install the Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Build the OP program
+        run: |
+          cd o1vm
+          make -C ./ethereum-optimism/op-program op-program
+          cd ..
+
+      - name: Start the local HTTP server
+        run: |
+          python -m http.server 8765 &
+
+      #
+      # Tests
+      #
+
+      - name: Execute o1vm in E2E flavor using cached data
+        run: |
+          eval $(opam env)
+          cd o1vm
+          unzip -q -o /tmp/o1vm-e2e-testing-cache.zip -d ./
+          RUN_WITH_CACHED_DATA="y" FILENAME="env-for-latest-l2-block.sh" O1VM_FLAVOR="pickles" STOP_AT="=3000000" ./run-code.sh

o1vm/Dockerfile-e2e-testing-cache

@@ -0,0 +1,12 @@
+FROM alpine
+
+# Add necessary utilities
+RUN apk add --no-cache shadow unzip zip
+
+# Copy the files from the host to the Docker image
+# Assuming the files and folder are located 
+# in the same directory as the current Dockerfile on the host
+COPY o1vm-e2e-testing-cache.zip /tmp/cache-source/o1vm-e2e-testing-cache.zip
+
+# Command to copy the files to mounted volume
+CMD ["/bin/sh", "-c", "cp -r /tmp/cache-source/* /tmp/cache/"]

o1vm/README.md

@@ -1,4 +1,4 @@
-## o1VM: a zero-knowledge virtual machine
+# o1VM: a zero-knowledge virtual machine
 
 This crate contains an implementation of different components used to build a
 zero-knowledge virtual machine. For now, the implementation is specialised for
@@ -54,7 +54,7 @@ gvm install go1.21
 gvm use go1.21s
 ```
 
-You also will need to install the [Foundry](https://getfoundry.sh/) toolkit 
+You also will need to install the [Foundry](https://getfoundry.sh/) toolkit
 in order to utilize applicaitons like `cast`.
 
 ```shell
@@ -64,18 +64,21 @@ foundryup
 You will also need to install jq with your favorite packet manager.
 
 eg. on Ubuntu
+
 ```shell
 sudo apt-get install jq
 ```
 
 ## Running the Optimism demo
 
 Start by initializing the submodules:
+
 ```bash
 git submodule init && git submodule update
 ```
 
 Create an executable `rpcs.sh` file like:
+
 ```bash
 #!/usr/bin/env bash
 export L1_RPC=http://xxxxxxxxx
@@ -86,29 +89,34 @@ export L1_BEACON_RPC=http://xxxxxxxxx
 
 If you just want to test the state transition between the latest finalized L2
 block and its predecessor:
+
 ```bash
 ./run-code.sh
 ```
 
 By default this will also create a script named `env-for-latest-l2-block.sh` with a
 snapshot of all the information that you need to rerun the same test again:
+
 ```bash
 FILENAME=env-for-latest-l2-block.sh bash run-code.sh
 ```
 
 Alternatively, you also have the option to test the state transition between a
 specific block and its predecessor:
+
 ```bash
 # Set -n to the desired block transition you want to test.
 ./setenv-for-l2-block.sh -n 12826645
 ```
 
 In this case, you can run the demo using the following format:
+
 ```bash
 FILENAME=env-for-l2-block-12826645.sh bash run-code.sh
 ```
 
 In either case, `run-code.sh` will:
+
 1. Generate the initial state.
 2. Execute the OP program.
 3. Execute the OP program through the Cannon MIPS VM.
@@ -117,6 +125,7 @@ In either case, `run-code.sh` will:
 ## Flavors
 
 Different versions/flavors of the o1vm are available.
+
 - [legacy](./src/legacy/mod.rs) - to be deprecated.
 - [pickles](./src/pickles/mod.rs) (currently the default)
 
@@ -126,6 +135,7 @@ environment variable `O1VM_FLAVOR`.
 ## Testing the preimage read
 
 Run:
+
 ```bash
 ./test_preimage_read.sh [OP_DB_DIRECTORY] [NETWORK_NAME]
 ```
@@ -134,3 +144,34 @@ The default value for `OP_DB_DIRECTORY` would be the one from
 `setenv-for-latest-l2-block.sh` if the parameter is omitted.
 
 The `NETWORK_NAME` defaults to `sepolia`.
+
+## Running the o1vm with cached data
+
+If you want to run the o1vm with cached data, you can use the following steps:
+
+- Make sure you have [Docker Engine](https://docs.docker.com/engine/install/) and [Python3](https://www.python.org/downloads/) installed on your machine.
+- Fetch the cached data by executing the following command (it might take some time):
+
+```shell
+./fetch-e2e-testing-cache.sh
+```
+
+- Start the simple HTTP server (in background or in another terminal session):
+
+```shell
+python3 -m http.server 8765 &
+```
+
+- Then run the o1vm with the following command:
+
+```shell
+RUN_WITH_CACHED_DATA="y" FILENAME="env-for-latest-l2-block.sh" O1VM_FLAVOR="pickles" STOP_AT="=3000000" ./run-code.sh
+```
+
+- Don't forget to stop the HTTP server after you are done.
+
+- You can clean the cached data by executing the following command:
+
+```shell
+./clear-e2e-testing-cache.sh
+```

o1vm/clear-e2e-testing-cache.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+START=$(date +%s)
+
+echo ""
+echo "Cleaning up the E2E testing cache file system..."
+echo ""
+rm -rf op-program-db-for-latest-l2-block \
+  env-for-latest-l2-block.sh \
+  snapshot-state-3000000.json \
+  state.json \
+  meta.json \
+  out.json \
+  cpu.pprof
+
+RUNTIME=$(($(date +%s) - START))
+echo ""
+echo "Execution time: ${RUNTIME} s"
+echo ""

o1vm/fetch-e2e-testing-cache.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+START=$(date +%s)
+
+echo ""
+echo "Fetching the E2E testing cache..."
+echo ""
+docker run --rm -it --name o1vm-e2e-testing-cache --pull=always -v ./:/tmp/cache o1labs/proof-systems:o1vm-e2e-testing-cache
+unzip -q -o o1vm-e2e-testing-cache.zip -d ./
+rm -rf o1vm-e2e-testing-cache.zip
+
+RUNTIME=$(($(date +%s) - START))
+echo ""
+echo "Execution time: ${RUNTIME} s"
+echo ""

o1vm/publish-e2e-testing-cache.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+START=$(date +%s)
+# ARCH="amd64"
+DOCKER_FILE_NAME="Dockerfile-e2e-testing-cache"
+DOCKER_HUB_USER_NAME="o1labs"
+DOCKER_HUB_REPO_NAME="proof-systems"
+DOCKER_HUB_IMAGE_TAG="o1vm-e2e-testing-cache"
+DOCKER_IMAGE_FULL_NAME="${DOCKER_HUB_USER_NAME}/${DOCKER_HUB_REPO_NAME}:${DOCKER_HUB_IMAGE_TAG}"
+
+echo ""
+echo "Preparing the file system..."
+echo ""
+zip -r -q o1vm-e2e-testing-cache.zip op-program-db-for-latest-l2-block \
+  env-for-latest-l2-block.sh \
+  snapshot-state-3000000.json \
+  state.json \
+  meta.json \
+  out.json \
+  cpu.pprof
+
+echo ""
+echo "Building the '${DOCKER_IMAGE_FULL_NAME}' Docker image..."
+echo ""
+docker rmi -f ${DOCKER_IMAGE_FULL_NAME} || true
+docker rmi -f ${DOCKER_HUB_IMAGE_TAG} || true
+# docker build --platform linux/${ARCH} -t ${DOCKER_HUB_IMAGE_TAG} -f ${DOCKER_FILE_NAME} .
+docker build -t ${DOCKER_HUB_IMAGE_TAG} -f ${DOCKER_FILE_NAME} .
+
+echo ""
+echo "Publishing the '${DOCKER_IMAGE_FULL_NAME}' Docker image..."
+echo ""
+docker tag ${DOCKER_HUB_IMAGE_TAG} ${DOCKER_IMAGE_FULL_NAME}
+docker push ${DOCKER_IMAGE_FULL_NAME}
+echo ""
+
+echo ""
+echo "Cleaning up the file system..."
+echo ""
+rm -rf o1vm-e2e-testing-cache.zip
+
+RUNTIME=$(($(date +%s) - START))
+echo ""
+echo "Execution time: ${RUNTIME} s"
+echo ""

o1vm/run-code.sh

@@ -10,6 +10,10 @@ set -u
 
 source $FILENAME
 
-./run-op-program.sh
-./run-cannon.sh
+if [ "${RUN_WITH_CACHED_DATA:-}" == "y" ]; then
+    echo "The Op-Program and the Cannon apps were not executed because the cached data usage was requested"
+else
+    ./run-op-program.sh
+    ./run-cannon.sh
+fi
 ./run-vm.sh