ROCm · dgaliffiAMD · Jul 11, 2024 · Jun 3, 2024 · Jul 1, 2024
@@ -1,6 +1,6 @@
 # MIT License
 #
-# Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -37,14 +37,21 @@ variables:
   #   argument unused during compilation: '--rtlib=compiler-rt'
   CXX_FLAGS: "-Wno-unused-command-line-argument -Wall -Wextra -Werror"
   HIP_FLAGS: "-Wno-unused-command-line-argument -Wall -Wextra -Werror"
+  # Keep in sync with ROCM_VERSION in Dockerfiles/hip-libraries-cuda-ubuntu.Dockerfile
+  # and Dockerfiles/hip-libraries-rocm-ubuntu.Dockerfile
+  DOCKER_ROCM_VERSION: 6.1.0
+  DOCKER_HIP_LIBRARIES_ROCM_TAG: rocm-ubuntu-${DOCKER_ROCM_VERSION}
+  DOCKER_HIP_LIBRARIES_CUDA_TAG: cuda-ubuntu-${DOCKER_ROCM_VERSION}
+  DOCKER_HIP_LIBRARIES_ROCM: $DOCKER_TAG_PREFIX:$DOCKER_HIP_LIBRARIES_ROCM_TAG
+  DOCKER_HIP_LIBRARIES_CUDA: $DOCKER_TAG_PREFIX:$DOCKER_HIP_LIBRARIES_CUDA_TAG
 
 stages:
   - lint
   - build
   - test
 
 clang-format:
-  image: $DOCKER_TAG_PREFIX:rocm-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_ROCM
   stage: lint
   needs: []
   tags:
@@ -60,7 +67,7 @@ clang-format:
     - Scripts/CodeFormat/check_format.sh $CI_MERGE_REQUEST_DIFF_BASE_SHA --binary "$CLANG_FORMAT"
 
 copyright-date:
-  image: $DOCKER_TAG_PREFIX:rocm-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_ROCM
   stage: lint
   needs: []
   tags:
@@ -88,7 +95,7 @@ copyright-date:
     - >-
       /kaniko/executor
       --context "${CI_PROJECT_DIR}/Dockerfiles"
-      --dockerfile "${CI_PROJECT_DIR}/Dockerfiles/hip-libraries-${TAG}.Dockerfile"
+      --dockerfile "${CI_PROJECT_DIR}/Dockerfiles/${DOCKERFILE}"
       --destination "docker.io/${DOCKER_TAG_PREFIX}:${TAG}"
       ${NO_PUSH}
   rules:
@@ -103,20 +110,22 @@ build:rocm-ubuntu-dockerfile:
   extends:
     - .build:dockerfiles
   variables:
-    TAG: rocm-ubuntu
+    DOCKERFILE: hip-libraries-rocm-ubuntu.Dockerfile
+    TAG: $DOCKER_HIP_LIBRARIES_ROCM_TAG
 
 build:cuda-ubuntu-dockerfile:
   extends:
     - .build:dockerfiles
   variables:
-    TAG: cuda-ubuntu
+    DOCKERFILE: hip-libraries-cuda-ubuntu.Dockerfile
+    TAG: $DOCKER_HIP_LIBRARIES_CUDA_TAG
 
 ########################
 # Ubuntu make          #
 ########################
 
 build:make-rocm:
-  image: $DOCKER_TAG_PREFIX:rocm-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_ROCM
   stage: build
   extends:
     - .rules:build
@@ -127,7 +136,7 @@ build:make-rocm:
     - cd $CI_PROJECT_DIR && make CXXFLAGS="$HIP_FLAGS" -j $(nproc)
 
 build:make-cuda:
-  image: $DOCKER_TAG_PREFIX:cuda-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_CUDA
   stage: build
   extends:
     - .rules:build
@@ -151,7 +160,7 @@ build:make-cuda:
       - $CI_PROJECT_DIR/build
 
 build:cmake-rocm:
-  image: $DOCKER_TAG_PREFIX:rocm-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_ROCM
   extends:
     - .build:cmake
     - .gpus:rocm-gpus
@@ -175,28 +184,28 @@ build:cmake-rocm:
     - cmake --install $CI_PROJECT_DIR/build --prefix $CI_PROJECT_DIR/install
 
 build:cmake-cuda:
- image: $DOCKER_TAG_PREFIX:cuda-ubuntu
- extends:
-   - .build:cmake
- tags:
-   - build
- script:
-   - cmake
-     -S $CI_PROJECT_DIR
-     -B $CI_PROJECT_DIR/build
-     -D GPU_RUNTIME=CUDA
-     -D CMAKE_CXX_FLAGS="$CXX_FLAGS"
-     -D CMAKE_CUDA_FLAGS="$CUDA_FLAGS"
-     -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip
-     2>&1 | tee cmake_log.txt
-   # check if all dependencies were found
-   - |-
-     if grep -qi "could not find" cmake_log.txt; then
-         echo "Some CMake libraries could not be found"
-         exit 1
-     fi
-   - cmake --build $CI_PROJECT_DIR/build
-   - cmake --install $CI_PROJECT_DIR/build --prefix $CI_PROJECT_DIR/install
+  image: $DOCKER_HIP_LIBRARIES_CUDA
+  extends:
+    - .build:cmake
+  tags:
+    - build
+  script:
+    - cmake
+      -S $CI_PROJECT_DIR
+      -B $CI_PROJECT_DIR/build
+      -D GPU_RUNTIME=CUDA
+      -D CMAKE_CXX_FLAGS="$CXX_FLAGS"
+      -D CMAKE_CUDA_FLAGS="$CUDA_FLAGS"
+      -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip
+      2>&1 | tee cmake_log.txt
+    # check if all dependencies were found
+    - |-
+      if grep -qi "could not find" cmake_log.txt; then
+          echo "Some CMake libraries could not be found"
+          exit 1
+      fi
+    - cmake --build $CI_PROJECT_DIR/build
+    - cmake --install $CI_PROJECT_DIR/build --prefix $CI_PROJECT_DIR/install
 
 ########################
 # Ubuntu Tests         #
@@ -207,18 +216,18 @@ build:cmake-cuda:
   extends:
     - .rules:test
   script:
-    - cd $CI_PROJECT_DIR/build && ctest --output-on-failure
+    - cd $CI_PROJECT_DIR/build && ctest --output-on-failure --parallel 8
 
 test:rocm:
-  image: $DOCKER_TAG_PREFIX:rocm-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_ROCM
   extends:
     - .test
     - .gpus:rocm
   needs:
     - build:cmake-rocm
 
 test:cuda:
-  image: $DOCKER_TAG_PREFIX:cuda-ubuntu
+  image: $DOCKER_HIP_LIBRARIES_CUDA
   extends:
     - .test
     - .gpus:nvcc
@@ -386,7 +395,7 @@ test:windows-nvcc-vs:
     # So for now, just add the library path here.
     - $env:PATH = "${env:HIP_PATH}\bin;" + $env:PATH
     - cd "$CI_PROJECT_DIR/build"
-    - ctest --output-on-failure --timeout 15
+    - ctest --output-on-failure --timeout 15 --parallel 8
     - cmake --install "$CI_PROJECT_DIR/build" --prefix "$CI_PROJECT_DIR/install"
   needs: []
 

@@ -1,7 +1,6 @@
-
 # MIGraphX - Torch Examples
 
-# Summary
+## Summary
 
 The examples in this subdirectory showcase the functionality for executing quantized models using MIGraphX. The Torch-MIGraphX integration library is used to achieve this, where PyTorch is used to quantize models, and MIGraphX is used to execute them on AMD GPUs.
 

@@ -6,7 +6,7 @@ This example walks through the dynamo Post Training Quantization (PTQ) workflow
 
 ## Prerequisites
 
-- You must follow the installation instructions for the torch_migraphx library in [README.md](README.md) before using this example.
+- You must follow the installation instructions for the torch_migraphx library in [AI/MIGraphX/Quantization](https://github.com/ROCm/rocm-examples/tree/develop/AI/MIGraphX/Quantization/) before using this example.
 
 ## Steps for running a quantized model using torch_migraphx
 

@@ -10,7 +10,7 @@ The examples in this subdirectory showcase several GPU-implementations of financ
 
 - [CMake](https://cmake.org/download/) (at least version 3.21)
 - OR GNU Make - available via the distribution's package manager
-- [ROCm](https://docs.amd.com/bundle/ROCm-Installation-Guide-v5.1.3/page/Overview_of_ROCm_Installation_Methods.html) (at least version 5.x.x)
+- [ROCm](https://rocm.docs.amd.com/projects/HIP/en/latest/install/install.html) (at least version 6.x.x)
 
 ### Windows
 

@@ -1,5 +1,13 @@
+# syntax=docker/dockerfile:latest
+# Above is required for substitutions in environment variables
+
 # CUDA based docker image
-FROM nvidia/cuda:12.0.0-devel-ubuntu20.04
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04
+
+# The ROCm versions that this image is based of.
+# Always write this down as major.minor.patch
+ENV ROCM_VERSION=6.1.0
+ENV ROCM_VERSION_APT=${ROCM_VERSION%.0}
 
 # Base packages that are required for the installation
 RUN export DEBIAN_FRONTEND=noninteractive; \
@@ -19,17 +27,17 @@ RUN export DEBIAN_FRONTEND=noninteractive; \
         vulkan-validationlayers \
         libglfw3-dev \
         gfortran \
-        # Nvidia driver version needed for hipSOLVER's CUDA backend.
-        # See https://docs.nvidia.com/deploy/cuda-compatibility/index.html#default-to-minor-version.
-        nvidia-driver-455 \
     && rm -rf /var/lib/apt/lists/*
 
-# Install HIP using the installer script
+# Install the HIP compiler and libraries from the ROCm repositories
 RUN export DEBIAN_FRONTEND=noninteractive; \
-    wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - \
-    && echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/6.0/ ubuntu main' > /etc/apt/sources.list.d/rocm.list \
+    mkdir -p /etc/apt/keyrings \
+    && wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg \
+    && echo "deb [arch=amd64, signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION_APT/ jammy main" > /etc/apt/sources.list.d/rocm.list \
+    && printf 'Package: *\nPin: origin "repo.radeon.com"\nPin-Priority: 9001\n' > /etc/apt/preferences.d/radeon.pref \
     && apt-get update -qq \
-    && apt-get install -y hip-base hipify-clang rocm-core hipcc hip-dev
+    && apt-get install -y hip-base hipify-clang rocm-core hipcc hip-dev rocm-llvm-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install CMake
 RUN wget https://github.com/Kitware/CMake/releases/download/v3.21.7/cmake-3.21.7-linux-x86_64.sh \
@@ -39,81 +47,87 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v3.21.7/cmake-3.21.7
 
 ENV PATH="/cmake/bin:/opt/rocm/bin:${PATH}"
 
-ENV HIP_COMPILER=nvcc HIP_PLATFORM=nvidia HIP_RUNTIME=cuda
-
 RUN echo "/opt/rocm/lib" >> /etc/ld.so.conf.d/rocm.conf \
     && ldconfig
 
+ENV HIP_COMPILER=nvcc HIP_PLATFORM=nvidia HIP_RUNTIME=cuda
+
 # Install rocRAND
-RUN wget https://github.com/ROCm/rocRAND/archive/refs/tags/rocm-6.0.0.tar.gz \
-    && tar -xf ./rocm-6.0.0.tar.gz \
-    && rm ./rocm-6.0.0.tar.gz \
-    && cmake -S ./rocRAND-rocm-6.0.0 -B ./rocRAND-rocm-6.0.0/build \
+RUN wget https://github.com/ROCm/rocRAND/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O rocrand.tar.gz \
+    && mkdir rocrand \
+    && tar -xf ./rocrand.tar.gz --strip-components 1 -C rocrand \
+    && rm ./rocrand.tar.gz \
+    && cmake -S ./rocrand -B ./rocrand/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D BUILD_HIPRAND=OFF \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
         -D NVGPU_TARGETS="50" \
-    && cmake --build ./rocRAND-rocm-6.0.0/build --target install \
-    && rm -rf ./rocRAND-rocm-6.0.0
+    && cmake --build ./rocrand/build --target install \
+    && rm -rf ./rocrand
 
 # Install hipCUB
-RUN wget https://github.com/ROCm/hipCUB/archive/refs/tags/rocm-6.0.0.tar.gz \
-    && tar -xf ./rocm-6.0.0.tar.gz \
-    && rm ./rocm-6.0.0.tar.gz \
-    && cmake -S ./hipCUB-rocm-6.0.0 -B ./hipCUB-rocm-6.0.0/build \
+RUN wget https://github.com/ROCm/hipCUB/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O hipcub.tar.gz \
+    && mkdir hipcub \
+    && tar -xf ./hipcub.tar.gz --strip-components 1 -C hipcub \
+    && rm ./hipcub.tar.gz \
+    && cmake -S ./hipcub -B ./hipcub/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
-    && cmake --build ./hipCUB-rocm-6.0.0/build --target install \
-    && rm -rf ./hipCUB-rocm-6.0.0
+    && cmake --build ./hipcub/build --target install \
+    && rm -rf ./hipcub
 
 # Install hipBLAS
-# hipBLAS cmake for rocm-6.0.0 is broken added CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ as fix
-RUN wget https://github.com/ROCm/hipBLAS/archive/refs/tags/rocm-6.0.0.tar.gz \
-    && tar -xf ./rocm-6.0.0.tar.gz \
-    && rm ./rocm-6.0.0.tar.gz \
-    && CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ cmake -S ./hipBLAS-rocm-6.0.0 -B ./hipBLAS-rocm-6.0.0/build \
+RUN wget https://github.com/ROCm/hipBLAS/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O hipblas.tar.gz \
+    && mkdir hipblas \
+    && tar -xf ./hipblas.tar.gz --strip-components 1 -C hipblas \
+    && rm ./hipblas.tar.gz \
+    && CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ cmake -S ./hipblas -B ./hipblas/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
         -D USE_CUDA=ON \
-    && cmake --build ./hipBLAS-rocm-6.0.0/build --target install \
-    && rm -rf ./hipBLAS-rocm-6.0.0
+    && cmake --build ./hipblas/build --target install \
+    && rm -rf ./hipblas
 
 # Install hipSOLVER
-# hipSOLVER cmake for rocm-6.0.0 is broken added CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ as fix
-RUN wget https://github.com/ROCm/hipSOLVER/archive/refs/tags/rocm-6.0.0.tar.gz \
-    && tar -xf ./rocm-6.0.0.tar.gz \
-    && rm ./rocm-6.0.0.tar.gz \
-    && CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ cmake -S ./hipSOLVER-rocm-6.0.0 -B ./hipSOLVER-rocm-6.0.0/build \
+# hipSOLVER cmake for rocm-6.1.0 is broken added CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ as fix
+RUN wget https://github.com/ROCm/hipSOLVER/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O hipsolver.tar.gz \
+    && mkdir hipsolver \
+    && tar -xf ./hipsolver.tar.gz --strip-components 1 -C hipsolver \
+    && rm ./hipsolver.tar.gz \
+    && CXXFLAGS=-D__HIP_PLATFORM_NVIDIA__ cmake -S ./hipsolver -B ./hipsolver/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
         -D USE_CUDA=ON \
-    && cmake --build ./hipSOLVER-rocm-6.0.0/build --target install \
-    && rm -rf ./hipSOLVER-rocm-6.0.0
+    && cmake --build ./hipsolver/build --target install \
+    && rm -rf ./hipsolver
 
 # Install hipRAND
-# Build from commit that removes deprecated macro use
-RUN git clone https://github.com/ROCm/hipRAND.git hipRAND-rocm-6.0.0 \
-    && cd hipRAND-rocm-6.0.0 \
-    && git reset --hard  4925f0da96fad5b9f532ddc79f1f52fc279d329f \
-    && cmake -S . -B ./build \
+# Manually replace usage of __HIP_PLATFORM_NVCC__ with __HIP_PLATFORM_NVIDIA__. See
+# https://github.com/ROCm/hipRAND/commit/4925f0da96fad5b9f532ddc79f1f52fc279d329f
+RUN wget https://github.com/ROCm/hipRAND/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O hiprand.tar.gz \
+    && mkdir hiprand \
+    && tar -xf ./hiprand.tar.gz --strip-components 1 -C hiprand \
+    && rm ./hiprand.tar.gz \
+    && sed -i s/__HIP_PLATFORM_NVCC__/__HIP_PLATFORM_NVIDIA__/ ./hiprand/library/include/hiprand/hiprand.h \
+    && cmake -S ./hiprand -B ./hiprand/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
         -D BUILD_WITH_LIB=CUDA \
         -D NVGPU_TARGETS="50" \
-    && cmake --build ./build --target install \
-    && cd .. \
-    && rm -rf ./hipRAND-rocm-6.0.0
+    && cmake --build ./hiprand/build --target install \
+    && rm -rf ./hiprand
 
 # Install hipFFT
-RUN wget https://github.com/ROCm/hipFFT/archive/refs/tags/rocm-6.0.0.tar.gz \
-    && tar -xf ./rocm-6.0.0.tar.gz \
-    && rm ./rocm-6.0.0.tar.gz \
-    && cmake -S ./hipFFT-rocm-6.0.0 -B ./hipFFT-rocm-6.0.0/build \
+RUN wget https://github.com/ROCm/hipFFT/archive/refs/tags/rocm-${ROCM_VERSION}.tar.gz -O hipfft.tar.gz \
+    && mkdir hipfft \
+    && tar -xf ./hipfft.tar.gz --strip-components 1 -C hipfft \
+    && rm ./hipfft.tar.gz \
+    && cmake -S ./hipfft -B ./hipfft/build \
         -D CMAKE_MODULE_PATH=/opt/rocm/lib/cmake/hip \
         -D CMAKE_INSTALL_PREFIX=/opt/rocm \
         -D BUILD_WITH_LIB=CUDA \
-    && cmake --build ./hipFFT-rocm-6.0.0/build --target install \
-    && rm -rf ./hipFFT-rocm-6.0.0
+    && cmake --build ./hipfft/build --target install \
+    && rm -rf ./hipfft
 
 # Use render group as an argument from user
 ARG GID=109