Adding GitHub Pipeline for TT-Forge (#54)

Adding GitHub Pipeline for TT-Forge

The main workflow runs using a Docker container and can execute tests on silicon. The Docker image has a prebuilt toolchain and all necessary dependencies.
Note: The Docker image is too large to be built on the GitHub runner; it must be built locally or on a self-hosted runner.

Changes Made:

Added Dockerfile and workflow for building the image
Added GitHub workflow for building and testing
Modified CMake: moved MLIR environment build to toolchain build
Updated pytest.ini: selected only valid tests
Removed GitLab CI
Removed previous GitHub pipeline

.github/Dockerfile

@@ -0,0 +1,76 @@
+FROM ubuntu:22.04
+SHELL ["/bin/bash", "-c"]
+
+# Set environment variables
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain
+ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain
+ENV PROJECT_NAME=tt-forge
+ARG GITHUB_TOKEN
+
+# Install dependencies
+RUN apt-get update && apt-get install -y \
+    software-properties-common \
+    build-essential \
+    python3-dev \
+    python3-venv \
+    python3-pip \
+    git \
+    git-lfs \
+    libhwloc-dev \
+    pandoc \
+    libtbb-dev \
+    libcapstone-dev \
+    pkg-config \
+    linux-tools-generic \
+    ninja-build \
+    wget \
+    cmake \
+    ccache \
+    doxygen \
+    libgtest-dev \
+    libgmock-dev \
+    graphviz \
+    patchelf \
+    libyaml-cpp-dev \
+    libboost-all-dev
+
+# Install clang 17
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod u+x llvm.sh && \
+    ./llvm.sh 17 && \
+    apt install -y libc++-17-dev libc++abi-17-dev && \
+    ln -s /usr/bin/clang-17 /usr/bin/clang && \
+    ln -s /usr/bin/clang++-17 /usr/bin/clang++
+
+# Install python packages
+RUN pip install cmake \
+    pytest
+
+# Create a directory for the build and toolchain
+ARG BUILD_DIR=/home/build
+RUN mkdir -p $BUILD_DIR && \
+    mkdir -p $TTMLIR_TOOLCHAIN_DIR && \
+    mkdir -p $TTFORGE_TOOLCHAIN_DIR
+
+# Clone the project and update submodules
+# Pass in PAT to clone private repositories
+RUN git clone https://[email protected]/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \
+    cd $BUILD_DIR/$PROJECT_NAME && \
+    git submodule update --init --recursive -f
+
+# Build the toolchain
+WORKDIR $BUILD_DIR/$PROJECT_NAME
+RUN source env/activate && \
+    cmake -B env/build env && \
+    cmake --build env/build
+
+# Build project to test the container
+RUN source env/activate && \
+    cmake -G Ninja -B build . && \
+    cmake --build build
+
+# Clean up the build directory
+RUN cd .. && \
+    rm -rf $BUILD_DIR/$PROJECT_NAME && \
+    unset GITHUB_TOKEN

.github/workflows/build-image.yml

@@ -0,0 +1,44 @@
+name: Build and Publish Docker Image
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: self-hosted # Github runners dont have enough storage space to build this image
+
+    steps:
+
+      - run: |
+          echo Pwd: $(pwd)
+          echo User: $(whoami)
+          ls -la /home/ubuntu/actions-runner/_work/tt-forge/tt-forge
+        shell: bash
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/Dockerfile
+          sparse-checkout-cone-mode: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .github
+          file: .github/Dockerfile
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:${{ github.sha }}
+            ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest

.github/workflows/docker-build.yml

@@ -0,0 +1,75 @@
+name: Build in Docker
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+
+  build-and-test:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+          - runs-on: self-hosted        
+
+    runs-on: ${{ matrix.build.runs-on }}
+
+    container:
+      image: ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest
+      options: --user root --device /dev/tenstorrent/0
+      volumes:
+        - /dev/hugepages:/dev/hugepages
+        - /dev/hugepages-1G:/dev/hugepages-1G 
+        - /etc/udev/rules.d:/etc/udev/rules.d 
+        - /lib/modules:/lib/modules
+        - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env
+    steps:
+    - uses: actions/checkout@v4
+      with:
+          submodules: recursive
+          fetch-depth: 0 # Fetch all history and tags
+
+    - name: Set reusable strings
+      id: strings
+      shell: bash
+      run: |
+        echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
+
+    - name: Git safe dir
+      run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
+
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        create-symlink: true
+        key: ${{ matrix.build.runs-on }}-runtime-${{ matrix.build.enable_runtime }}-${{ env.SDK_VERSION }}
+
+    - name: Build
+      shell: bash
+      run: |
+        source env/activate
+        cmake -G Ninja -B build .
+        cmake --build build
+
+    - name: Run Test
+      shell: bash
+      run: |
+        source env/activate
+        pytest
+      continue-on-error: true
+
+    - name: Upload Test Report
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-reports-${{ matrix.build.runs-on }}
+        path: reports/report.xml
+
+    - name: Show Test Report
+      uses: mikepenz/action-junit-report@v4
+      if: success() || failure()
+      with:
+        report_paths: reports/report.xml
+        check_name: TT-Forge Tests

.github/workflows/on-pr.yml

@@ -0,0 +1,11 @@
+name: On PR
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  docker-build:
+    uses: ./.github/workflows/docker-build.yml
+    secrets: inherit

.github/workflows/on-push.yml

@@ -0,0 +1,11 @@
+name: On push
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  docker-build:
+    uses: ./.github/workflows/docker-build.yml
+    secrets: inherit

.gitignore

@@ -53,3 +53,6 @@ build_deps/
 # ClangD
 compile_commands.json
 \n\n# Exclude LFS files to keep the public repo small
+
+# Test reports
+reports/report.xml