test_e2eshark.yml

# Copyright 2024 Advanced Micro Devices
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: E2ESHARK Test Suite
on:
  pull_request:
  workflow_dispatch:

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit). The workflow name is prepended to avoid conflicts between
  # different workflows.
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  torch-mlir:
    strategy:
      matrix:
        version: [3.11]
        os: [nodai-amdgpu-w7900-x86-64]
    runs-on: ${{matrix.os}}
    # steps:
    #   - name: Checkout repo
    #     uses: actions/checkout@v2
    #     with:
    #       submodules: true
    #       repository: llvm/torch-mlir
    #       path: torch-mlir
      
    #   - name: "Setting up Python"
    #     uses: actions/setup-python@v5
    #     with:
    #       python-version: ${{matrix.version}}

    #   - name: Build torch-mlir
    #     run: |
    #       git submodule update --init --progres
    #       python -m venv mlir_venv
    #       source mlir_venv/bin/activate
    #       pip install --upgrade pip
    #       pip install -r ./externals/llvm-project/mlir/python/requirements.txt
    #       pip install -r requirements.txt
    #       pip install -r torchvision-requirements.txt
    #       pip install "pybind11[global]"
    #       cmake -S "$repo_root/externals/llvm-project/llvm" -B "$build_dir" \
    #         -GNinja \
    #         -DCMAKE_BUILD_TYPE=Release \
    #         -DPython3_FIND_VIRTUALENV=ONLY \
    #         -DLLVM_ENABLE_PROJECTS=mlir \
    #         -DLLVM_EXTERNAL_PROJECTS="torch-mlir" \
    #         -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$PWD" \
    #         -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
    #         -DLLVM_TARGETS_TO_BUILD=host
    #       cmake --build build
    #       CMAKE_GENERATOR=Ninja python setup.py bdist_wheel --dist-dir ./torch-mlir-wheel -v
    #       pip uninstall torch-mlir
    #       pip install torch-mlir-wheel/torch_mlir-0.0.1-cp311-cp311-linux_x86_64.whl
    #       deactivate
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
        with:
          submodules: true
          repository: saienduri/torch-mlir-fork
          path: torch-mlir

      - name: Install python deps (nightly)
        run: |
          bash build_tools/ci/install_python_deps.sh nightly
        working-directory: ./torch-mlir

      - name: Build project and wheels
        run: |
          bash build_tools/ci/build_posix.sh
          bash build_tools/python/build_python_wheels.sh
        working-directory: ./torch-mlir
  
  # iree:
  #   strategy
  #     matrix:
  #       version: [3.11]
  #       os: [nodai-amdgpu-w7900-x86-64]
  #   runs-on: ${{matrix.os}}
  #   env:
  #     IREE_VENV_DIR: ${{ github.workspace }}/iree_venv
  #   steps:
  #     - name: Checkout repo
  #       uses: actions/checkout@v2
  #       with:
  #         repository: openxla/iree
  #         path: iree
      
  #     - name: "Setting up Python"
  #       uses: actions/setup-python@v5
  #       with:
  #         python-version: ${{matrix.version}}

  #     - name: "Setup Python venv"
  #       run: python -m venv ${IREE_VENV_DIR}

  #     - name: Build iree
  #       run: |
  #         source ${IREE_VENV_DIR}/bin/activate
  #         git submodule foreach git config --get remote.origin.fetch
  #         pip install --upgrade pip
  #         pip install -r runtime/bindings/python/iree/runtime/build_requirements.txt
  #         cmake -G Ninja -B ../iree-build/ -S . \
  #           -DCMAKE_BUILD_TYPE=RelWithDebInfo \
  #           -DIREE_ENABLE_ASSERTIONS=ON \
  #           -DIREE_ENABLE_SPLIT_DWARF=ON \
  #           -DIREE_ENABLE_THIN_ARCHIVES=ON \
  #           -DCMAKE_C_COMPILER=clang \
  #           -DCMAKE_CXX_COMPILER=clang++ \
  #           -DIREE_ENABLE_LLD=ON
  #         cmake --build ../iree-build/
  #       working-directory: ./iree
  
  turbine:
    strategy:
      matrix:
        os: [nodai-amdgpu-w7900-x86-64]
    runs-on: ${{matrix.os}}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-Turbine
          path: turbine
  
  e2eshark:
    needs: [turbine, torch-mlir]
    strategy:
      matrix:
        version: [3.11]
        os: [nodai-amdgpu-w7900-x86-64]
    runs-on: ${{matrix.os}}
    env:
      E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-TestSuite
          path: test-suite
      
      - name: "Setting up Python"
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.version}}

      - name: "Setup Python venv"
        run: python -m venv ${E2E_VENV_DIR}

      - name: Build
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          git submodule update --init
          pip install --upgrade pip
          pip install -r /home/esaimana/torch-mlir/requirements.txt
          pip install -r /home/esaimana/torch-mlir/torchvision-requirements.txt
          pip install torch-mlir -f https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels
          pip install -r ./e2eshark/requirements.txt
          pip install -e ../turbine/core[testing]
          pip install -e ../turbine/models
        working-directory: ./test-suite

      - name: Run Tests
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          cd e2eshark
          python ./run.py -c ../../torch-mlir/build --report --cachedir ~/.cache/huggingface --mode turbine --tests pytorch/models/beit-base-patch16-224-pt22k-ft22k pytorch/models/bge-base-en-v1.5 pytorch/models/mit-b0 pytorch/models/bert-large-uncased pytorch/models/deit-small-distilled-patch16-224 --cleanup
        working-directory: ./test-suite