From e8a80cb6fe27aa0978c0ed001a9c13b1b66cef87 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 22 Jul 2024 14:00:20 -0700 Subject: [PATCH 1/3] Use ubuntu gh runners for Docker builds --- .github/workflows/build-conda-images.yml | 2 +- .github/workflows/build-libtorch-images.yml | 4 ++-- .github/workflows/build-manywheel-images.yml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-conda-images.yml b/.github/workflows/build-conda-images.yml index c977c8c5c..7b689e2c7 100644 --- a/.github/workflows/build-conda-images.yml +++ b/.github/workflows/build-conda-images.yml @@ -28,7 +28,7 @@ env: jobs: build-docker: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: cuda_version: ["11.8", "12.1", "12.4", "cpu"] diff --git a/.github/workflows/build-libtorch-images.yml b/.github/workflows/build-libtorch-images.yml index f066cf834..7fbaa48f3 100644 --- a/.github/workflows/build-libtorch-images.yml +++ b/.github/workflows/build-libtorch-images.yml @@ -30,7 +30,7 @@ env: jobs: build-docker-cuda: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -49,7 +49,7 @@ jobs: run: | libtorch/build_docker.sh build-docker-rocm: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: rocm_version: ["6.0", "6.1"] diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index aaf420d60..2d103687c 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -40,7 +40,7 @@ env: jobs: build-docker-cuda: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -62,7 +62,7 @@ jobs: manywheel/build_docker.sh # NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649 build-docker-cuda-manylinux_2_28: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -102,7 +102,7 @@ jobs: run: | manywheel/build_docker.sh build-docker-rocm: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest strategy: matrix: rocm_version: ["6.0", "6.1"] @@ -194,7 +194,7 @@ jobs: run: | manywheel/build_docker.sh build-docker-xpu: - runs-on: linux.12xlarge.ephemeral + runs-on: ubuntu-latest env: GPU_ARCH_TYPE: xpu steps: From 415b7a20e08738c1832a06fd91a27d5f3b7301d0 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 23 Jul 2024 13:46:01 -0700 Subject: [PATCH 2/3] test --- .github/workflows/build-conda-images.yml | 2 +- .github/workflows/build-libtorch-images.yml | 4 +- .github/workflows/build-manywheel-images.yml | 8 +-- .github/workflows/build-nccl-linux.yml | 58 ++++++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-nccl-linux.yml diff --git a/.github/workflows/build-conda-images.yml b/.github/workflows/build-conda-images.yml index 7b689e2c7..76aca0f23 100644 --- a/.github/workflows/build-conda-images.yml +++ b/.github/workflows/build-conda-images.yml @@ -28,7 +28,7 @@ env: jobs: build-docker: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: cuda_version: ["11.8", "12.1", "12.4", "cpu"] diff --git a/.github/workflows/build-libtorch-images.yml b/.github/workflows/build-libtorch-images.yml index 7fbaa48f3..ef13904ed 100644 --- a/.github/workflows/build-libtorch-images.yml +++ b/.github/workflows/build-libtorch-images.yml @@ -30,7 +30,7 @@ env: jobs: build-docker-cuda: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -49,7 +49,7 @@ jobs: run: | libtorch/build_docker.sh build-docker-rocm: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: rocm_version: ["6.0", "6.1"] diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml index 2d103687c..810205f7b 100644 --- a/.github/workflows/build-manywheel-images.yml +++ b/.github/workflows/build-manywheel-images.yml @@ -40,7 +40,7 @@ env: jobs: build-docker-cuda: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -62,7 +62,7 @@ jobs: manywheel/build_docker.sh # NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649 build-docker-cuda-manylinux_2_28: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: cuda_version: ["12.4", "12.1", "11.8"] @@ -102,7 +102,7 @@ jobs: run: | manywheel/build_docker.sh build-docker-rocm: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral strategy: matrix: rocm_version: ["6.0", "6.1"] @@ -194,7 +194,7 @@ jobs: run: | manywheel/build_docker.sh build-docker-xpu: - runs-on: ubuntu-latest + runs-on: linux.9xlarge.ephemeral env: GPU_ARCH_TYPE: xpu steps: diff --git a/.github/workflows/build-nccl-linux.yml b/.github/workflows/build-nccl-linux.yml new file mode 100644 index 000000000..3fd13771a --- /dev/null +++ b/.github/workflows/build-nccl-linux.yml @@ -0,0 +1,58 @@ +name: build-nccl-magma + +on: + push: + branches: + main + paths: + - common/install_conda.sh + - .github/workflows/build-nccl-linux.yml + pull_request: + paths: + + - common/install_conda.sh + - .github/workflows/build-nccl-linux.yml + +# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179 +defaults: + run: + shell: bash -x -e -l {0} +env: + BUILD_ENVIRONMENT: build-linux-magma + IN_CI: 1 + IS_GHA: 1 + +jobs: + build-linux-magma: + runs-on: linux.2xlarge + strategy: + matrix: + cuda_version: ["121", "118"] + steps: + - name: Checkout PyTorch builder + uses: actions/checkout@v3 + - name: Build Magma Cuda + working-directory: magma + run: | + make magma-cuda${{ matrix.cuda_version }} + - name: Save as artifact + uses: actions/upload-artifact@v2 + with: + path: magma/output/linux-64/magma-cuda*.bz2 + - name: Install conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.8 + auto-update-conda: true + miniconda-version: "latest" + activate-environment: build-linux-magma + - name: Conda install anaconda-client + shell: bash -l {0} + run: | + conda install -y conda-build anaconda-client + - name: Push MAGMA to anaconda + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + anaconda --token $ANACONDA_TOKEN upload -u pytorch --force magma/output/linux-64/magma-cuda*.bz2 + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} From a24f61ba28c34897911ad8158db1910e4f99ecdc Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 23 Jul 2024 13:47:08 -0700 Subject: [PATCH 3/3] test --- .github/workflows/build-nccl-linux.yml | 58 -------------------------- 1 file changed, 58 deletions(-) delete mode 100644 .github/workflows/build-nccl-linux.yml diff --git a/.github/workflows/build-nccl-linux.yml b/.github/workflows/build-nccl-linux.yml deleted file mode 100644 index 3fd13771a..000000000 --- a/.github/workflows/build-nccl-linux.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: build-nccl-magma - -on: - push: - branches: - main - paths: - - common/install_conda.sh - - .github/workflows/build-nccl-linux.yml - pull_request: - paths: - - - common/install_conda.sh - - .github/workflows/build-nccl-linux.yml - -# For setup-miniconda, see https://github.com/conda-incubator/setup-miniconda/issues/179 -defaults: - run: - shell: bash -x -e -l {0} -env: - BUILD_ENVIRONMENT: build-linux-magma - IN_CI: 1 - IS_GHA: 1 - -jobs: - build-linux-magma: - runs-on: linux.2xlarge - strategy: - matrix: - cuda_version: ["121", "118"] - steps: - - name: Checkout PyTorch builder - uses: actions/checkout@v3 - - name: Build Magma Cuda - working-directory: magma - run: | - make magma-cuda${{ matrix.cuda_version }} - - name: Save as artifact - uses: actions/upload-artifact@v2 - with: - path: magma/output/linux-64/magma-cuda*.bz2 - - name: Install conda - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: 3.8 - auto-update-conda: true - miniconda-version: "latest" - activate-environment: build-linux-magma - - name: Conda install anaconda-client - shell: bash -l {0} - run: | - conda install -y conda-build anaconda-client - - name: Push MAGMA to anaconda - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - run: | - anaconda --token $ANACONDA_TOKEN upload -u pytorch --force magma/output/linux-64/magma-cuda*.bz2 - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}