From 3e39e127caddc9ad2e870637d095f0920e67c1db Mon Sep 17 00:00:00 2001
From: Yeuoly <45712896+Yeuoly@users.noreply.github.com>
Date: Wed, 16 Oct 2024 18:05:40 +0800
Subject: [PATCH] refactor: using docker to execute CI tests on Github actions
 to prevent environmental breaking changes (#98)

---
 .github/workflows/tests-amd64.yml        | 58 ---------------------
 .github/workflows/tests-arm64.yml        | 60 ----------------------
 .github/workflows/tests.yml              | 35 +++++++++++++
 docker/amd64-test/dockerfile             | 64 ++++++++++++++++++++++++
 docker/amd64/dockerfile                  |  2 +-
 docker/arm64-test/dockerfile             | 64 ++++++++++++++++++++++++
 docker/arm64/dockerfile                  |  6 +--
 tests/integration_tests/conf/config.yaml |  2 -
 8 files changed, 167 insertions(+), 124 deletions(-)
 delete mode 100644 .github/workflows/tests-amd64.yml
 delete mode 100644 .github/workflows/tests-arm64.yml
 create mode 100644 .github/workflows/tests.yml
 create mode 100644 docker/amd64-test/dockerfile
 create mode 100644 docker/arm64-test/dockerfile

diff --git a/.github/workflows/tests-amd64.yml b/.github/workflows/tests-amd64.yml
deleted file mode 100644
index 975762a..0000000
--- a/.github/workflows/tests-amd64.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: Run GoTest
-
-on:
-  pull_request:
-    branches:
-      - main
-  push:
-    branches:
-      - main
-
-concurrency:
-  group: test-amd64-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: Test AMD64
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install System Dependencies
-        run: sudo apt-get install -y pkg-config gcc libseccomp-dev
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.20.6
-
-      - name: Install dependencies
-        run: go mod tidy
-
-      - name: Compile library
-        run: bash ./build/build_amd64.sh
-
-      - name: Setup Nodejs20.11.1
-        uses: actions/setup-node@v3
-        with:
-          node-version: '20.11.1'
-
-      - name: Link Nodejs
-        run: sudo ln -sf "$(which node)" /usr/local/bin/node
-
-      - name: Setup Python3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
-
-      - name: Link Python3.10
-        run: sudo ln -sf "$(which python3.10)" /usr/local/bin/python3
-
-      - name: Install Python dependencies
-        run: pip install httpx requests jinja2
-
-      - name: Run Intgeration tests
-        run: sudo go test -timeout 120s -v ./tests/integration_tests/...
diff --git a/.github/workflows/tests-arm64.yml b/.github/workflows/tests-arm64.yml
deleted file mode 100644
index 1510ffd..0000000
--- a/.github/workflows/tests-arm64.yml
+++ /dev/null
@@ -1,60 +0,0 @@
-name: Run GoTest
-
-on:
-  pull_request:
-    branches:
-      - main
-  push:
-    branches:
-      - main
-
-concurrency:
-  group: test-arm64-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: Test ARM64
-    runs-on: arm64_runner
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Install System Dependencies
-        run: sudo apt-get install -y pkg-config gcc libseccomp-dev
-
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.20.6
-
-      - name: Install dependencies
-        run: go mod tidy
-
-      - name: Compile library
-        run: bash ./build/build_arm64.sh
-
-      - name: Setup Nodejs20.11.1
-        uses: actions/setup-node@v3
-        with:
-          node-version: '20.11.1'
-
-      - name: Setup Python3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
-
-      - name: Link Python3.10
-        run: sudo ln -sf "$(which python3.10)" /usr/local/bin/python3
-
-      - name: Install Python dependencies
-        run: pip install httpx requests jinja2
-
-      - name: Link Go
-        run: sudo ln -s "$(which go)" /usr/local/bin/go
-
-      - name: Run Intgeration tests
-        run: sudo go test -timeout 120s -v ./tests/integration_tests/...
-        env:
-          PYTHON_PATH: /usr/bin/python3.10
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..b6c156d
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,35 @@
+name: Run GoTest
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: test-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Test
+    runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm64_runner' }}
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build and run tests
+        run: |
+          if [ "${{ matrix.arch }}" == "amd64" ]; then
+            docker build -t test -f docker/amd64-test/dockerfile .
+            docker run --rm test
+          elif [ "${{ matrix.arch }}" == "arm64" ]; then
+            docker build -t test -f docker/arm64-test/dockerfile .
+            docker run --rm test
+          fi
diff --git a/docker/amd64-test/dockerfile b/docker/amd64-test/dockerfile
new file mode 100644
index 0000000..da4260c
--- /dev/null
+++ b/docker/amd64-test/dockerfile
@@ -0,0 +1,64 @@
+FROM golang:1.20.6 AS builder
+
+COPY . /app
+WORKDIR /app
+
+# if you located in China, you can use aliyun mirror to speed up
+# && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list
+
+# install dependencies and build binary
+RUN apt-get update && apt-get install -y pkg-config gcc libseccomp-dev && go mod tidy && bash ./build/build_amd64.sh
+
+FROM python:3.10-slim-bookworm as tester
+
+# if you located in China, you can use aliyun mirror to speed up
+# && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list
+
+# install system dependencies
+RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+       pkg-config \
+       libseccomp-dev \
+       wget \
+       curl \
+       xz-utils \
+       zlib1g=1:1.3.dfsg+really1.3.1-1 \
+       expat=2.6.3-1 \
+       perl=5.38.2-5 \
+       libsqlite3-0=3.46.0-1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# workdir
+WORKDIR /app
+
+# checkout
+COPY . /app
+
+# copy binary and env from builder
+COPY --from=builder /app/internal/core/runner/python/python.so /app/internal/core/runner/python/python.so
+COPY --from=builder /app/internal/core/runner/nodejs/nodejs.so /app/internal/core/runner/nodejs/nodejs.so
+
+# copy test config file
+COPY conf/config.yaml /conf/config.yaml
+# copy python dependencies
+COPY dependencies/python-requirements.txt /dependencies/python-requirements.txt
+
+# install python dependencies
+RUN pip3 install --no-cache-dir httpx==0.27.2 requests==2.32.3 jinja2==3.0.3 PySocks httpx[socks]
+
+# install node
+RUN wget -O /opt/node-v20.11.1-linux-x64.tar.xz https://npmmirror.com/mirrors/node/v20.11.1/node-v20.11.1-linux-x64.tar.xz \
+    && tar -xvf /opt/node-v20.11.1-linux-x64.tar.xz -C /opt \
+    && ln -s /opt/node-v20.11.1-linux-x64/bin/node /usr/local/bin/node \
+    && rm -f /opt/node-v20.11.1-linux-x64.tar.xz
+
+# install golang 1.20.6
+RUN wget https://golang.org/dl/go1.20.6.linux-amd64.tar.gz \
+    && tar -C /usr/local -xzf go1.20.6.linux-amd64.tar.gz \
+    && ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && rm -f go1.20.6.linux-amd64.tar.gz
+
+# run test
+RUN go test -timeout 120s -v ./tests/integration_tests/...
diff --git a/docker/amd64/dockerfile b/docker/amd64/dockerfile
index 9cd157e..b217e0a 100644
--- a/docker/amd64/dockerfile
+++ b/docker/amd64/dockerfile
@@ -28,7 +28,7 @@ COPY conf/config.yaml /conf/config.yaml
 COPY dependencies/python-requirements.txt /dependencies/python-requirements.txt
 
 RUN chmod +x /main /env \
-    && pip3 install --no-cache-dir jinja2 requests httpx PySocks httpx[socks] \
+    && pip3 install --no-cache-dir httpx==0.27.2 requests==2.32.3 jinja2==3.0.3 PySocks httpx[socks] \
     && wget -O /opt/node-v20.11.1-linux-x64.tar.xz https://npmmirror.com/mirrors/node/v20.11.1/node-v20.11.1-linux-x64.tar.xz \
     && tar -xvf /opt/node-v20.11.1-linux-x64.tar.xz -C /opt \
     && ln -s /opt/node-v20.11.1-linux-x64/bin/node /usr/local/bin/node \
diff --git a/docker/arm64-test/dockerfile b/docker/arm64-test/dockerfile
new file mode 100644
index 0000000..7ebd4b6
--- /dev/null
+++ b/docker/arm64-test/dockerfile
@@ -0,0 +1,64 @@
+FROM golang:1.20.6 AS builder
+
+COPY . /app
+WORKDIR /app
+
+# if you located in China, you can use aliyun mirror to speed up
+# && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list
+
+# install dependencies and build binary
+RUN apt-get update && apt-get install -y pkg-config gcc libseccomp-dev && go mod tidy && bash ./build/build_arm64.sh
+
+FROM python:3.10-slim-bookworm as tester
+
+# if you located in China, you can use aliyun mirror to speed up
+# && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list
+
+# install system dependencies
+RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+       pkg-config \
+       libseccomp-dev \
+       wget \
+       curl \
+       xz-utils \
+       zlib1g=1:1.3.dfsg+really1.3.1-1 \
+       expat=2.6.3-1 \
+       perl=5.38.2-5 \
+       libsqlite3-0=3.46.0-1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# workdir
+WORKDIR /app
+
+# checkout
+COPY . /app
+
+# copy binary and env from builder
+COPY --from=builder /app/internal/core/runner/python/python.so /app/internal/core/runner/python/python.so
+COPY --from=builder /app/internal/core/runner/nodejs/nodejs.so /app/internal/core/runner/nodejs/nodejs.so
+
+# copy test config file
+COPY conf/config.yaml /conf/config.yaml
+# copy python dependencies
+COPY dependencies/python-requirements.txt /dependencies/python-requirements.txt
+
+# install python dependencies
+RUN pip3 install --no-cache-dir httpx==0.27.2 requests==2.32.3 jinja2==3.0.3 PySocks httpx[socks]
+
+# install node
+RUN wget -O /opt/node-v20.11.1-linux-arm64.tar.xz https://npmmirror.com/mirrors/node/v20.11.1/node-v20.11.1-linux-arm64.tar.xz \
+    && tar -xvf /opt/node-v20.11.1-linux-arm64.tar.xz -C /opt \
+    && ln -s /opt/node-v20.11.1-linux-arm64/bin/node /usr/local/bin/node \
+    && rm -f /opt/node-v20.11.1-linux-arm64.tar.xz
+
+# install golang 1.20.6
+RUN wget https://golang.org/dl/go1.20.6.linux-arm64.tar.gz \
+    && tar -C /usr/local -xzf go1.20.6.linux-arm64.tar.gz \
+    && ln -s /usr/local/go/bin/go /usr/local/bin/go \
+    && rm -f go1.20.6.linux-arm64.tar.gz
+
+# run test
+RUN go test -timeout 120s -v ./tests/integration_tests/...
diff --git a/docker/arm64/dockerfile b/docker/arm64/dockerfile
index c82a4e6..cb98794 100644
--- a/docker/arm64/dockerfile
+++ b/docker/arm64/dockerfile
@@ -4,7 +4,7 @@ FROM python:3.10-slim-bookworm
 # && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list
 RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
-    && apt-get install -y --no-install-recommends
+    && apt-get install -y --no-install-recommends \
        pkg-config \
        libseccomp-dev \
        wget \
@@ -14,7 +14,7 @@ RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list
        expat=2.6.3-1 \
        perl=5.38.2-5 \
        libsqlite3-0=3.46.0-1 \
-    && apt-get clean
+    && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 # copy main binary to /main
@@ -28,7 +28,7 @@ COPY conf/config.yaml /conf/config.yaml
 COPY dependencies/python-requirements.txt /dependencies/python-requirements.txt
 
 RUN chmod +x /main /env \
-    && pip3 install --no-cache-dir jinja2 requests httpx PySocks httpx[socks] \
+    && pip3 install --no-cache-dir httpx==0.27.2 requests==2.32.3 jinja2==3.0.3 PySocks httpx[socks] \
     && wget -O /opt/node-v20.11.1-linux-arm64.tar.xz https://npmmirror.com/mirrors/node/v20.11.1/node-v20.11.1-linux-arm64.tar.xz \
     && tar -xvf /opt/node-v20.11.1-linux-arm64.tar.xz -C /opt \
     && ln -s /opt/node-v20.11.1-linux-arm64/bin/node /usr/local/bin/node \
diff --git a/tests/integration_tests/conf/config.yaml b/tests/integration_tests/conf/config.yaml
index 7f2d556..f30b53d 100644
--- a/tests/integration_tests/conf/config.yaml
+++ b/tests/integration_tests/conf/config.yaml
@@ -10,8 +10,6 @@ python_lib_path:
   - "/usr/local/lib/python3.10"
   - "/usr/lib/python3.10"
   - "/usr/lib/python3"
-  - "/opt/hostedtoolcache/Python/3.10.14/x64/lib/python3.10"
-  - "/opt/hostedtoolcache/Python/3.10.15/arm64/lib/python3.10"
   - "/usr/lib/x86_64-linux-gnu"
   - "/usr/lib/aarch64-linux-gnu"
   - "/etc/ssl/certs/ca-certificates.crt"