From aca9be6d30e3038c26a45a31d1874ec2c839de4a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 16:36:01 -0500
Subject: [PATCH 001/395] Add simple workflow that just runs nvidia-smi.

---
 .github/test.yml | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/test.yml

diff --git a/.github/test.yml b/.github/test.yml
new file mode 100644
index 00000000000..ba8de5a07dc
--- /dev/null
+++ b/.github/test.yml
@@ -0,0 +1,31 @@
+name: CI (GPU)
+
+on:
+  push:
+    branches:
+      - main
+      - "pull-request/[0-9]+"
+
+concurrency:
+  group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  build-gpu:
+    if: github.repository == 'NVIDIA/stdexec'
+    name: GPU (${{ matrix.name }}, CUDA ${{ matrix.cuda }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { name: "clang 16",   cuda: "12.0", cxx: "clang++", tag: "llvm16-cuda12.0", gpu: "v100", sm: "70", driver: "latest", arch: "amd64" }
+    runs-on: linux-${{ matrix.arch }}-gpu-${{ matrix.gpu }}-${{ matrix.driver }}-1
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.04-cpp-${{ matrix.tag }}-ubuntu22.04
+      env:
+        CUDA_VERSION: "${{ matrix.cuda }}"
+        NVIDIA_VISIBLE_DEVICES: "${{ env.NVIDIA_VISIBLE_DEVICES }}"
+    steps:
+      - name: Run nvidia-smi to make sure GPU is working
+        run: nvidia-smi
\ No newline at end of file

From a1e39b75cc6b3c1abed04eaeb33d774a74ef2973 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 16:44:51 -0500
Subject: [PATCH 002/395] lol, it needs to be in a workflows folder.

---
 .github/{ => workflows}/test.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/{ => workflows}/test.yml (100%)

diff --git a/.github/test.yml b/.github/workflows/test.yml
similarity index 100%
rename from .github/test.yml
rename to .github/workflows/test.yml

From a4ba910d2bba8a830f53eea3eae47dd9d1f5fa2f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 16:45:25 -0500
Subject: [PATCH 003/395] Fix repository name.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ba8de5a07dc..c6b6a05f7e0 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   build-gpu:
-    if: github.repository == 'NVIDIA/stdexec'
+    if: github.repository == 'NVIDIA/cccl'
     name: GPU (${{ matrix.name }}, CUDA ${{ matrix.cuda }})
     strategy:
       fail-fast: false

From d7a60a11519e3869549d4b39248a62d1419ea0d4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 16:48:35 -0500
Subject: [PATCH 004/395] Add ops-bot file.

---
 .github/workflows/ops-bot.yaml | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 .github/workflows/ops-bot.yaml

diff --git a/.github/workflows/ops-bot.yaml b/.github/workflows/ops-bot.yaml
new file mode 100644
index 00000000000..794526b9a06
--- /dev/null
+++ b/.github/workflows/ops-bot.yaml
@@ -0,0 +1,4 @@
+# This file controls which features from the `ops-bot` repository below are enabled.
+# - https://github.com/rapidsai/ops-bot
+
+copy_prs: true
\ No newline at end of file

From c4b4ec43ab9036a9c0f4085cc3038d3f16610029 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 16:54:51 -0500
Subject: [PATCH 005/395] Remove redundant ops-bot file.

---
 .github/workflows/ops-bot.yaml | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100644 .github/workflows/ops-bot.yaml

diff --git a/.github/workflows/ops-bot.yaml b/.github/workflows/ops-bot.yaml
deleted file mode 100644
index 794526b9a06..00000000000
--- a/.github/workflows/ops-bot.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# This file controls which features from the `ops-bot` repository below are enabled.
-# - https://github.com/rapidsai/ops-bot
-
-copy_prs: true
\ No newline at end of file

From c052036308910398d7ca8f9f455e2dcf2319de95 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 28 Apr 2023 17:05:31 -0500
Subject: [PATCH 006/395] Update .github/workflows/test.yml

Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com>
---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c6b6a05f7e0..e7c8f058e37 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: linux-${{ matrix.arch }}-gpu-${{ matrix.gpu }}-${{ matrix.driver }}-1
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.04-cpp-${{ matrix.tag }}-ubuntu22.04
+      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.tag }}-ubuntu22.04
       env:
         CUDA_VERSION: "${{ matrix.cuda }}"
         NVIDIA_VISIBLE_DEVICES: "${{ env.NVIDIA_VISIBLE_DEVICES }}"

From 63333b915eae968e1dc65d0cb67b0d1edf1d8d03 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 11:22:15 -0500
Subject: [PATCH 007/395] Add matrix for test gcc versions.

---
 .github/workflows/test.yml | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e7c8f058e37..a59dd8ea41b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,21 +11,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-gpu:
+  build:
     if: github.repository == 'NVIDIA/cccl'
-    name: GPU (${{ matrix.name }}, CUDA ${{ matrix.cuda }})
+    name: Build (${{ matrix.cpu }}, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }}, CUDA ${{ matrix.ctk }})
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - { name: "clang 16",   cuda: "12.0", cxx: "clang++", tag: "llvm16-cuda12.0", gpu: "v100", sm: "70", driver: "latest", arch: "amd64" }
-    runs-on: linux-${{ matrix.arch }}-gpu-${{ matrix.gpu }}-${{ matrix.driver }}-1
+        gcc: [7, 8, 9, 10]
+        ctk: [11, 12.1]
+        cxx: [11, 14, 17]
+        cpu: [amd64, arm64] 
+    runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.tag }}-ubuntu22.04
-      env:
-        CUDA_VERSION: "${{ matrix.cuda }}"
-        NVIDIA_VISIBLE_DEVICES: "${{ env.NVIDIA_VISIBLE_DEVICES }}"
+      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda${{ matrix.ctk }}-ubuntu20.04
     steps:
-      - name: Run nvidia-smi to make sure GPU is working
-        run: nvidia-smi
\ No newline at end of file
+      - name: Echo matrix
+        run: echo ${{ matrix.gcc }}, ${{matrix.ctk}}, ${{matrix.cxx}}, ${{matrix.cpu}}
\ No newline at end of file

From 98db32aaaef72d5d68ca58223f2c313a730a5548 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 11:23:40 -0500
Subject: [PATCH 008/395] Set max parallelism to 8.

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a59dd8ea41b..aea625ff4b7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,6 +16,7 @@ jobs:
     name: Build (${{ matrix.cpu }}, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }}, CUDA ${{ matrix.ctk }})
     strategy:
       fail-fast: false
+      max-parallel: 8
       matrix:
         gcc: [7, 8, 9, 10]
         ctk: [11, 12.1]

From eab8c4f4f17be336503a9c8c422e59a9c466ad4c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 11:26:19 -0500
Subject: [PATCH 009/395] CUDA 11 -> CUDA 11.0.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index aea625ff4b7..360c5269072 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,7 +19,7 @@ jobs:
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9, 10]
-        ctk: [11, 12.1]
+        ctk: [11.0, 12.1]
         cxx: [11, 14, 17]
         cpu: [amd64, arm64] 
     runs-on: linux-${{ matrix.cpu }}-cpu4

From 088b1bee540ccb78fadb62d260e28bd9ef1cc31d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 11:31:10 -0500
Subject: [PATCH 010/395] Disable ARM.

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 360c5269072..fe304be2e2a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,8 +21,9 @@ jobs:
         gcc: [7, 8, 9, 10]
         ctk: [11.0, 12.1]
         cxx: [11, 14, 17]
-        cpu: [amd64, arm64] 
-    runs-on: linux-${{ matrix.cpu }}-cpu4
+        #cpu: [amd64, arm64] 
+    #runs-on: linux-${{ matrix.cpu }}-cpu4
+    runs-on: linux-amd64-cpu4
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda${{ matrix.ctk }}-ubuntu20.04

From 582d9772d950f6ef4046959667f3d0db3bd91cb7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 11:41:28 -0500
Subject: [PATCH 011/395] Use quotes around ctk version.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fe304be2e2a..81ed7198f82 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,7 +19,7 @@ jobs:
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9, 10]
-        ctk: [11.0, 12.1]
+        ctk: ["11.0", "12.1"]
         cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4

From 15fd6d1ba2546c6e78e234546a25d3ff0b3b72e6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:20:38 -0500
Subject: [PATCH 012/395] Make matrix specific to CUDA 12.1.

---
 .github/workflows/test.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 81ed7198f82..c4a066ef72a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,22 +11,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
+  cuda12.1:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (${{ matrix.cpu }}, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }}, CUDA ${{ matrix.ctk }})
+    name: Build (CUDA 12.1, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9, 10]
-        ctk: ["11.0", "12.1"]
         cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4
     runs-on: linux-amd64-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda${{ matrix.ctk }}-ubuntu20.04
+      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda12.1-ubuntu20.04
     steps:
       - name: Echo matrix
-        run: echo ${{ matrix.gcc }}, ${{matrix.ctk}}, ${{matrix.cxx}}, ${{matrix.cpu}}
\ No newline at end of file
+        run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
\ No newline at end of file

From 459e38b0abf879ab65951053945bdc8d40ce0994 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:21:37 -0500
Subject: [PATCH 013/395] Change cuda12.1 to cuda12_1.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c4a066ef72a..bd129b01c19 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  cuda12.1:
+  cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA 12.1, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }})
     strategy:

From c854ae11040a17a2b3f75dad0fe3ef4a273ff010 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:26:53 -0500
Subject: [PATCH 014/395] Add matrix for CUDA 11.

---
 .github/workflows/test.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bd129b01c19..727701f8ccf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,6 +26,24 @@ jobs:
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda12.1-ubuntu20.04
+    steps:
+      - name: Echo matrix
+        run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
+  cuda11_0:
+    if: github.repository == 'NVIDIA/cccl'
+    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        gcc: [7, 8, 9]
+        cxx: [11, 14, 17]
+        #cpu: [amd64, arm64] 
+    #runs-on: linux-${{ matrix.cpu }}-cpu4
+    runs-on: linux-amd64-cpu4
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda11.0-ubuntu18.04
     steps:
       - name: Echo matrix
         run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
\ No newline at end of file

From cd9857f9f9c04a2b3b05774af48f9f778aeb9d08 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:29:15 -0500
Subject: [PATCH 015/395] Disable cxx in matrix to eliminate extraneous jobs.

---
 .github/workflows/test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 727701f8ccf..419fb56ad91 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,13 +13,13 @@ concurrency:
 jobs:
   cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 12.1, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }})
+    name: Build (CUDA 12.1, gcc ${{ matrix.gcc }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9, 10]
-        cxx: [11, 14, 17]
+        #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4
     runs-on: linux-amd64-cpu4
@@ -31,13 +31,13 @@ jobs:
         run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }}, C++${{ matrix.cxx }})
+    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9]
-        cxx: [11, 14, 17]
+        #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4
     runs-on: linux-amd64-cpu4
@@ -46,4 +46,4 @@ jobs:
       image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda11.0-ubuntu18.04
     steps:
       - name: Echo matrix
-        run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
\ No newline at end of file
+        run: echo "gcc ${{ matrix.gcc }}"
\ No newline at end of file

From 2b27cfedc55becf96fb52f019875b45eace295c3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:34:05 -0500
Subject: [PATCH 016/395] Parameterize ubuntu version.

---
 .github/workflows/test.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 419fb56ad91..888f95a7c26 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,37 +13,38 @@ concurrency:
 jobs:
   cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 12.1, gcc ${{ matrix.gcc }})
+    name: Build (CUDA 12.1, ${{ matrix.gcc }}, Ubuntu ${{ matrix.ubuntu }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
-        gcc: [7, 8, 9, 10]
+        ubuntu: ["20.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4
     runs-on: linux-amd64-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda12.1-ubuntu20.04
+      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
         run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }})
+    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }}, Ubuntu ${{ matrix.ubuntu }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
         gcc: [7, 8, 9]
+        ubuntu: ["18.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
     #runs-on: linux-${{ matrix.cpu }}-cpu4
     runs-on: linux-amd64-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda11.0-ubuntu18.04
+      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda11.0-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
         run: echo "gcc ${{ matrix.gcc }}"
\ No newline at end of file

From 4107148d62bd21c9b9113e4f8a37f1fbf9f82ed1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:36:59 -0500
Subject: [PATCH 017/395] Change gcc matrix list to compiler and add clang
 versions.

---
 .github/workflows/test.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 888f95a7c26..2e93f804556 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,11 +13,12 @@ concurrency:
 jobs:
   cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 12.1, ${{ matrix.gcc }}, Ubuntu ${{ matrix.ubuntu }})
+    name: Build (CUDA 12.1, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
+        compiler: [gcc7, gcc8, gcc9, gcc10, clang9, clang10, clang11, clang12, clang13, clang14]
         ubuntu: ["20.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
@@ -25,10 +26,10 @@ jobs:
     runs-on: linux-amd64-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
-        run: echo "gcc ${{ matrix.gcc }}, std ${{matrix.cxx}}"
+        run: echo "${{matrix.compiler}}"
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA 11.0, gcc ${{ matrix.gcc }}, Ubuntu ${{ matrix.ubuntu }})
@@ -36,7 +37,7 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix:
-        gcc: [7, 8, 9]
+        compiler: [gcc7, gcc8, gcc9, clang9]
         ubuntu: ["18.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
@@ -44,7 +45,7 @@ jobs:
     runs-on: linux-amd64-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc${{ matrix.gcc }}-cuda11.0-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler}}-cuda11.0-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
-        run: echo "gcc ${{ matrix.gcc }}"
\ No newline at end of file
+        run: echo "${{matrix.compiler}}"
\ No newline at end of file

From c666fb0f191d504f0805ed93a69d0b2e24f5f5b7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:38:14 -0500
Subject: [PATCH 018/395] Fix matrix.gcc to matrix.compiler.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2e93f804556..9bdbaa2276c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
         run: echo "${{matrix.compiler}}"
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 11.0, gcc ${{ matrix.gcc }}, Ubuntu ${{ matrix.ubuntu }})
+    name: Build (CUDA 11.0, ${{matrix.compiler}}, Ubuntu ${{ matrix.ubuntu }})
     strategy:
       fail-fast: false
       max-parallel: 8

From 1acf2fb193cd98a39e57938d188f621606d8df69 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:45:18 -0500
Subject: [PATCH 019/395] Change clangN to llvmN to match devcontainer format.

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9bdbaa2276c..980d1816cc3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix:
-        compiler: [gcc7, gcc8, gcc9, gcc10, clang9, clang10, clang11, clang12, clang13, clang14]
+        compiler: [gcc7, gcc8, gcc9, gcc10, llvm9, llvm10, llvm11, llvm12, llvm13, llvm14]
         ubuntu: ["20.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 
@@ -37,7 +37,7 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix:
-        compiler: [gcc7, gcc8, gcc9, clang9]
+        compiler: [gcc7, gcc8, gcc9, llvm9]
         ubuntu: ["18.04"]
         #cxx: [11, 14, 17]
         #cpu: [amd64, arm64] 

From 9b827283a612d54a2b0eb6d097efe8f9122273c7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:45:44 -0500
Subject: [PATCH 020/395] Add CPU arch as single element list.

---
 .github/workflows/test.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 980d1816cc3..6fcec8b5f11 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,7 +13,7 @@ concurrency:
 jobs:
   cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 12.1, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }})
+    name: Build (CUDA 12.1, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
@@ -21,9 +21,9 @@ jobs:
         compiler: [gcc7, gcc8, gcc9, gcc10, llvm9, llvm10, llvm11, llvm12, llvm13, llvm14]
         ubuntu: ["20.04"]
         #cxx: [11, 14, 17]
-        #cpu: [amd64, arm64] 
-    #runs-on: linux-${{ matrix.cpu }}-cpu4
-    runs-on: linux-amd64-cpu4
+        cpu: [amd64] #[amd64, arm64] 
+        include:
+    runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
@@ -32,7 +32,7 @@ jobs:
         run: echo "${{matrix.compiler}}"
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 11.0, ${{matrix.compiler}}, Ubuntu ${{ matrix.ubuntu }})
+    name: Build (CUDA 11.0, ${{matrix.compiler}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
@@ -40,9 +40,8 @@ jobs:
         compiler: [gcc7, gcc8, gcc9, llvm9]
         ubuntu: ["18.04"]
         #cxx: [11, 14, 17]
-        #cpu: [amd64, arm64] 
-    #runs-on: linux-${{ matrix.cpu }}-cpu4
-    runs-on: linux-amd64-cpu4
+        cpu: [amd64] #[amd64, arm64] 
+    runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler}}-cuda11.0-ubuntu${{ matrix.ubuntu }}

From 429b2a4ee6fc694ee6f983691850e2bb99d2bb46 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:45:55 -0500
Subject: [PATCH 021/395] Add 22.04 builds.

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6fcec8b5f11..9bc8317be82 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -23,6 +23,14 @@ jobs:
         #cxx: [11, 14, 17]
         cpu: [amd64] #[amd64, arm64] 
         include:
+          - compiler: gcc11
+            ubuntu: "22.04"
+          - compiler: gcc12
+            ubuntu: "22.04"
+          - compiler: llvm15
+            ubuntu: "22.04"
+          - compiler: llvm16
+            ubuntu: "22.04"
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From 992dfbb0e4302aec72c7abf3976dc8bde8184140 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 12:55:05 -0500
Subject: [PATCH 022/395] Add comments.

---
 .github/workflows/test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9bc8317be82..d0fa90aa47f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,6 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Build configurations for CUDA 12.1
   cuda12_1:
     if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA 12.1, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
@@ -22,6 +23,7 @@ jobs:
         ubuntu: ["20.04"]
         #cxx: [11, 14, 17]
         cpu: [amd64] #[amd64, arm64] 
+        # Add configurations for newer gcc/clang that require Ubuntu 22.04
         include:
           - compiler: gcc11
             ubuntu: "22.04"
@@ -38,6 +40,7 @@ jobs:
     steps:
       - name: Echo matrix
         run: echo "${{matrix.compiler}}"
+  # Build configurations for CUDA 11.0
   cuda11_0:
     if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA 11.0, ${{matrix.compiler}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})

From 987caaae58270ea397e424cb742a0703bf966cde Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 13:43:51 -0500
Subject: [PATCH 023/395] manually specify the matrix.

---
 .github/workflows/test.yml | 40 +++++---------------------------------
 1 file changed, 5 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d0fa90aa47f..78c39086d7d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -11,51 +11,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # Build configurations for CUDA 12.1
-  cuda12_1:
+  build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 12.1, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
       matrix:
-        compiler: [gcc7, gcc8, gcc9, gcc10, llvm9, llvm10, llvm11, llvm12, llvm13, llvm14]
-        ubuntu: ["20.04"]
-        #cxx: [11, 14, 17]
-        cpu: [amd64] #[amd64, arm64] 
-        # Add configurations for newer gcc/clang that require Ubuntu 22.04
         include:
-          - compiler: gcc11
-            ubuntu: "22.04"
-          - compiler: gcc12
-            ubuntu: "22.04"
-          - compiler: llvm15
-            ubuntu: "22.04"
-          - compiler: llvm16
-            ubuntu: "22.04"
+          - cuda: '12.1' ubuntu: '20.04' cpu: 'amd64' compiler: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']
+          - cuda: '12.1' ubuntu: '22.04' cpu: 'amd64' compiler: ['gcc11', 'gcc12', 'llvm15', 'llvm16']
+          - cuda: '11.0' ubuntu: '18.04' cpu: 'amd64' compiler: ['gcc7', 'gcc8', 'gcc9', 'llvm9']
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
-    steps:
-      - name: Echo matrix
-        run: echo "${{matrix.compiler}}"
-  # Build configurations for CUDA 11.0
-  cuda11_0:
-    if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA 11.0, ${{matrix.compiler}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix:
-        compiler: [gcc7, gcc8, gcc9, llvm9]
-        ubuntu: ["18.04"]
-        #cxx: [11, 14, 17]
-        cpu: [amd64] #[amd64, arm64] 
-    runs-on: linux-${{ matrix.cpu }}-cpu4
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler}}-cuda11.0-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
         run: echo "${{matrix.compiler}}"
\ No newline at end of file

From c2ff1e009230ae9199d713acd6c79b789f5a8532 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 13:44:22 -0500
Subject: [PATCH 024/395] Correct hardcoded cuda version to use from matrix.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 78c39086d7d..4044d387a3d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,7 +25,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda12.1-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Echo matrix
         run: echo "${{matrix.compiler}}"
\ No newline at end of file

From 8c063588dc1248eb43c5aab47fbedcd2911cdeda Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 13:46:04 -0500
Subject: [PATCH 025/395] Fix YAML syntax.

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4044d387a3d..7cae067376c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,9 +19,9 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          - cuda: '12.1' ubuntu: '20.04' cpu: 'amd64' compiler: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']
-          - cuda: '12.1' ubuntu: '22.04' cpu: 'amd64' compiler: ['gcc11', 'gcc12', 'llvm15', 'llvm16']
-          - cuda: '11.0' ubuntu: '18.04' cpu: 'amd64' compiler: ['gcc7', 'gcc8', 'gcc9', 'llvm9']
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: ['gcc11', 'gcc12', 'llvm15', 'llvm16']}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: ['gcc7', 'gcc8', 'gcc9', 'llvm9']}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From 52f5fb523637f45e4dcb5aa982055f90fc32a39f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:06:57 -0500
Subject: [PATCH 026/395] Try yaml anchors.

---
 .github/workflows/test.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7cae067376c..007943d10a5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,9 +19,15 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: ['gcc11', 'gcc12', 'llvm15', 'llvm16']}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: ['gcc7', 'gcc8', 'gcc9', 'llvm9']}
+          - &gcc-llvm-compilers
+            compilers: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']
+          - &new-compilers
+            compilers: ['gcc11', 'gcc12', 'llvm15', 'llvm16']
+          - &gcc-llvm-old-compilers
+            compilers: ['gcc7', 'gcc8', 'gcc9', 'llvm9']
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', <<: *gcc-llvm-compilers}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', <<: *new-compilers}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', <<: *gcc-llvm-old-compilers}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From ec83975fbcaa2ad31664fbefa043d804f22b3da7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:09:10 -0500
Subject: [PATCH 027/395] Anchors aren't supported.

---
 .github/workflows/test.yml | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 007943d10a5..2af610a859f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,15 +19,27 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          - &gcc-llvm-compilers
-            compilers: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'llvm9', 'llvm10', 'llvm11', 'llvm12', 'llvm13', 'llvm14']
-          - &new-compilers
-            compilers: ['gcc11', 'gcc12', 'llvm15', 'llvm16']
-          - &gcc-llvm-old-compilers
-            compilers: ['gcc7', 'gcc8', 'gcc9', 'llvm9']
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', <<: *gcc-llvm-compilers}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', <<: *new-compilers}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', <<: *gcc-llvm-old-compilers}
+          # Ubuntu 20.04
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
+          # Ubuntu 22.04
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
+          # Ubuntu 18.04
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From 8c57581d4b1ef1d054dbcb8b7859dbdd19b2636f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:15:04 -0500
Subject: [PATCH 028/395] Try it with nesting.

---
 .github/workflows/test.yml | 51 ++++++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2af610a859f..cce786184ee 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,27 +19,36 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          # Ubuntu 20.04
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
-          # Ubuntu 22.04
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
-          # Ubuntu 18.04
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
+          - cuda: '12.1'
+            ubuntu: '20.04'
+            cpu: 'amd64'
+            compilers:
+              - 'gcc7'
+              - 'gcc8'
+              - 'gcc9'
+              - 'gcc10'
+              - 'llvm9'
+              - 'llvm10'
+              - 'llvm11'
+              - 'llvm12'
+              - 'llvm13'
+              - 'llvm14'
+          - cuda: '12.1'
+            ubuntu: '22.04'
+            cpu: 'amd64'
+            compilers:
+              - 'gcc11'
+              - 'gcc12'
+              - 'llvm15'
+              - 'llvm16'
+          - cuda: '11.0'
+            ubuntu: '18.04'
+            cpu: 'amd64'
+            compilers:
+              - 'gcc7'
+              - 'gcc8'
+              - 'gcc9'
+              - 'llvm9'
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From dc12f527e44df084851b2be7863f40c1359f9ced Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:21:11 -0500
Subject: [PATCH 029/395] Nesting doesn't work. Manual it is.

---
 .github/workflows/test.yml | 51 ++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index cce786184ee..2af610a859f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,36 +19,27 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          - cuda: '12.1'
-            ubuntu: '20.04'
-            cpu: 'amd64'
-            compilers:
-              - 'gcc7'
-              - 'gcc8'
-              - 'gcc9'
-              - 'gcc10'
-              - 'llvm9'
-              - 'llvm10'
-              - 'llvm11'
-              - 'llvm12'
-              - 'llvm13'
-              - 'llvm14'
-          - cuda: '12.1'
-            ubuntu: '22.04'
-            cpu: 'amd64'
-            compilers:
-              - 'gcc11'
-              - 'gcc12'
-              - 'llvm15'
-              - 'llvm16'
-          - cuda: '11.0'
-            ubuntu: '18.04'
-            cpu: 'amd64'
-            compilers:
-              - 'gcc7'
-              - 'gcc8'
-              - 'gcc9'
-              - 'llvm9'
+          # Ubuntu 20.04
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
+          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
+          # Ubuntu 22.04
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
+          # Ubuntu 18.04
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From 53bbbc2776162ddc17b98ffe3236ae5661e1fa83 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:25:29 -0500
Subject: [PATCH 030/395] Better organization.

---
 .github/workflows/test.yml | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2af610a859f..8d9e46f665b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,27 +19,26 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          # Ubuntu 20.04
+          # gcc
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
+          # clang
+          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
           - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
-          # Ubuntu 22.04
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
           - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
           - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
-          # Ubuntu 18.04
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root

From 0eeaaabc1fa1c2951f9feab50a391ea3e0b0c9e1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:34:47 -0500
Subject: [PATCH 031/395] Add step to build thrust tests.

---
 .github/workflows/test.yml | 38 +++++++++++++++++++-------------------
 ci/test_thrust.sh          | 16 ++++++++++++++++
 2 files changed, 35 insertions(+), 19 deletions(-)
 create mode 100644 ci/test_thrust.sh

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8d9e46f665b..546c134d44a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,29 +20,29 @@ jobs:
       matrix:
         include:
           # gcc
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
+          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
+          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
+          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
+          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
           - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
           # clang
-          - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
-          - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
+          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
+          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
+          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
+          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
     runs-on: linux-${{ matrix.cpu }}-cpu4
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
-      - name: Echo matrix
-        run: echo "${{matrix.compiler}}"
\ No newline at end of file
+      - name: Build Thrust tests
+        run: ./ci/test_thrust.sh
\ No newline at end of file
diff --git a/ci/test_thrust.sh b/ci/test_thrust.sh
new file mode 100644
index 00000000000..32d14355bbf
--- /dev/null
+++ b/ci/test_thrust.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Clone Thrust repository
+git clone https://github.com/NVIDIA/thrust.git
+
+# Change to the Thrust directory
+cd thrust
+
+# Build Thrust tests
+mkdir build
+cd build
+cmake ..
+make
+
+# Run Thrust tests
+ctest --output-on-failure

From 820a98214ae3dc2ba55039ce2f55b6fbf4bfe15f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:36:31 -0500
Subject: [PATCH 032/395] Modify script to only build thrust tests.

---
 .github/workflows/test.yml             | 2 +-
 ci/{test_thrust.sh => build_thrust.sh} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename ci/{test_thrust.sh => build_thrust.sh} (79%)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 546c134d44a..a2059c2381a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -45,4 +45,4 @@ jobs:
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Build Thrust tests
-        run: ./ci/test_thrust.sh
\ No newline at end of file
+        run: ./ci/build_thrust.sh
\ No newline at end of file
diff --git a/ci/test_thrust.sh b/ci/build_thrust.sh
similarity index 79%
rename from ci/test_thrust.sh
rename to ci/build_thrust.sh
index 32d14355bbf..50608169a87 100644
--- a/ci/test_thrust.sh
+++ b/ci/build_thrust.sh
@@ -9,8 +9,8 @@ cd thrust
 # Build Thrust tests
 mkdir build
 cd build
-cmake ..
-make
+cmake -G ninja ..
+ninja
 
 # Run Thrust tests
-ctest --output-on-failure
+#ctest --output-on-failure

From 4d498f462a39e208254e30b1ac6d84b44fdb4e7f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:49:47 -0500
Subject: [PATCH 033/395] pwd

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a2059c2381a..8525a4b5f95 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -44,5 +44,7 @@ jobs:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
+      - name: Print working dir
+        run: pwd
       - name: Build Thrust tests
         run: ./ci/build_thrust.sh
\ No newline at end of file

From f4122f4ba105f1fbde04890c7c5455f31cdb6676 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:52:25 -0500
Subject: [PATCH 034/395] ls

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8525a4b5f95..e5c44a6a523 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -46,5 +46,7 @@ jobs:
     steps:
       - name: Print working dir
         run: pwd
+      - name: list
+        run: ls
       - name: Build Thrust tests
         run: ./ci/build_thrust.sh
\ No newline at end of file

From fc64641b74427da210a4fa040a5dcdfa38f35697 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 14:57:43 -0500
Subject: [PATCH 035/395] Add checkout step.

---
 .github/workflows/test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e5c44a6a523..741c1a42e6a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -44,6 +44,9 @@ jobs:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
       - name: Print working dir
         run: pwd
       - name: list

From 3968718832133331c7f9c09ae61f32a7755ea497 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:05:19 -0500
Subject: [PATCH 036/395] Explicitly run script with bash.

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 741c1a42e6a..3506cb7c53b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -47,9 +47,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          persist-credentials: false
       - name: Print working dir
         run: pwd
       - name: list
         run: ls
       - name: Build Thrust tests
-        run: ./ci/build_thrust.sh
\ No newline at end of file
+        run: bash ./ci/build_thrust.sh
\ No newline at end of file

From 283404e2537bb10adae7c2156a3c388bfcdefac1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:13:03 -0500
Subject: [PATCH 037/395] Correct ninja generator.

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 50608169a87..e572fad6036 100644
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -9,7 +9,7 @@ cd thrust
 # Build Thrust tests
 mkdir build
 cd build
-cmake -G ninja ..
+cmake -GNinja ..
 ninja
 
 # Run Thrust tests

From a77105a6e0cde3fb159ba4d88bebcf3afdad23d3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:13:24 -0500
Subject: [PATCH 038/395] Use 16 core machine.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3506cb7c53b..a06a7048ce1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -39,7 +39,7 @@ jobs:
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
           #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
           #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
-    runs-on: linux-${{ matrix.cpu }}-cpu4
+    runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}

From 5086c4c8c197e49d2ec11a700d8ac190ed24561e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:15:39 -0500
Subject: [PATCH 039/395] Only build thrust tests for sm70.

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index e572fad6036..1b03085f2b8 100644
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -9,7 +9,7 @@ cd thrust
 # Build Thrust tests
 mkdir build
 cd build
-cmake -GNinja ..
+cmake -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON ..
 ninja
 
 # Run Thrust tests

From ea2cd1bfd0b9d149a163b66e6b4bcfa639ce0b71 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:18:15 -0500
Subject: [PATCH 040/395] Use Paul's excellent cmake suggestions.

---
 ci/build_thrust.sh | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 1b03085f2b8..bcb1c79f86d 100644
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -3,14 +3,12 @@
 # Clone Thrust repository
 git clone https://github.com/NVIDIA/thrust.git
 
-# Change to the Thrust directory
-cd thrust
+# Configure Thrust
+cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON 
 
 # Build Thrust tests
-mkdir build
-cd build
-cmake -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON ..
-ninja
+cmake --build thrust/build
+
 
 # Run Thrust tests
 #ctest --output-on-failure

From 252dc0e35a4137bd81d5d43c56fdecc6242cb8ba Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 15:23:40 -0500
Subject: [PATCH 041/395] Need to recursively clone thrust to get CUB. Also use
 shallow clone.

---
 ci/build_thrust.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index bcb1c79f86d..c8241dc919d 100644
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 
-# Clone Thrust repository
-git clone https://github.com/NVIDIA/thrust.git
+
+# Clone Thrust repository 
+git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
 cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON 

From 1fcc3e37dddbb85255b0a8d3aff64e46b3382338 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 16:07:56 -0500
Subject: [PATCH 042/395] Add running Thrust tests using upload/download
 artifact between build/test.

---
 .github/workflows/test.yml | 23 ++++++++++++++++++++++-
 ci/build_thrust.sh         |  1 -
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a06a7048ce1..39faa86f0c1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,4 +53,25 @@ jobs:
       - name: list
         run: ls
       - name: Build Thrust tests
-        run: bash ./ci/build_thrust.sh
\ No newline at end of file
+        run: bash ./ci/build_thrust.sh
+      - name: Upload Thrust build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: thrust-build-artifacts
+          path: ./thrust/build/
+  test:
+    name: Run tests
+    needs: build
+    runs-on: linux-amd64-gpu-v100-latest-1
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - name: Download Thrust build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: thrust-build-artifacts
+      - name: Run Thrust tests
+        run: ctest --test-dir build --output-on-failure
\ No newline at end of file
diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index c8241dc919d..37312b2b2ff 100644
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 # Clone Thrust repository 
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 

From 0bc0a329ae9dc7f7f820a37e257b9e1037380122 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 17:20:56 -0500
Subject: [PATCH 043/395] print working directory and list

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 39faa86f0c1..3f902cb50b5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -74,4 +74,4 @@ jobs:
         with:
           name: thrust-build-artifacts
       - name: Run Thrust tests
-        run: ctest --test-dir build --output-on-failure
\ No newline at end of file
+        run: pwd && ls && ctest --test-dir build --output-on-failure
\ No newline at end of file

From 52257ae90fe08032a23ee076277040ae23e230a5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 17:21:39 -0500
Subject: [PATCH 044/395] Add parameter to script.

---
 ci/build_thrust.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 ci/build_thrust.sh

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
old mode 100644
new mode 100755
index 37312b2b2ff..4a93d064e5c
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,10 +1,19 @@
 #!/bin/bash
 
+# Require a compiler to be specified
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <CXX_COMPILER>"
+    exit 1
+fi
+
+# First argument is the CXX compiler
+CXX_COMPILER=$1
+
 # Clone Thrust repository 
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
-cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON 
+cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 # Build Thrust tests
 cmake --build thrust/build

From b8dd97d94fae422476e461b24fb7b24a0bd760fa Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 17:21:53 -0500
Subject: [PATCH 045/395] Add compiler name/version.

---
 .github/workflows/test.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3f902cb50b5..6d985f36840 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
           #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc12'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12'}
           # clang
           #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
@@ -42,16 +42,12 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{ matrix.compiler }}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
           persist-credentials: false
-      - name: Print working dir
-        run: pwd
-      - name: list
-        run: ls
       - name: Build Thrust tests
         run: bash ./ci/build_thrust.sh
       - name: Upload Thrust build artifacts

From 51cc6b113988c9f66ae1b8d0d690780f4a6ead4a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 18:01:43 -0500
Subject: [PATCH 046/395] Fix syntax.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6d985f36840..1a703b55f36 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
           #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12'}
+          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12'}}
           # clang
           #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
           #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}

From f23e935a918a97535f6a100e605f3038ea86475f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 18:04:04 -0500
Subject: [PATCH 047/395] For testing purposes, only configure, don't build

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 4a93d064e5c..45850ca23c9 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -16,7 +16,7 @@ git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 # Build Thrust tests
-cmake --build thrust/build
+#cmake --build thrust/build
 
 
 # Run Thrust tests

From 5af871d8cecb6145e086ffcfc74099f6d96b1ca9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 18:41:55 -0500
Subject: [PATCH 048/395] Disable explicit compiler.

---
 ci/build_thrust.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 45850ca23c9..30a30759792 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,19 +1,20 @@
 #!/bin/bash
 
 # Require a compiler to be specified
-if [ $# -ne 1 ]; then
-    echo "Usage: $0 <CXX_COMPILER>"
-    exit 1
-fi
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
 
 # First argument is the CXX compiler
-CXX_COMPILER=$1
+#CXX_COMPILER=$1
+
 
 # Clone Thrust repository 
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
-cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 # Build Thrust tests
 #cmake --build thrust/build

From 7749b346d1edf54bbeec0370b3941f05ecdca5a2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 18:49:46 -0500
Subject: [PATCH 049/395] Download build artifacts to build directory.

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1a703b55f36..2dbce674ce6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -69,5 +69,6 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: thrust-build-artifacts
+          path: build
       - name: Run Thrust tests
         run: pwd && ls && ctest --test-dir build --output-on-failure
\ No newline at end of file

From 6e6a82ad37cf711555de74fd4ee821482e03dba2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 18:56:55 -0500
Subject: [PATCH 050/395] Re-enable building Thrust.

---
 ci/build_thrust.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 30a30759792..1afe75d01b6 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -9,7 +9,6 @@
 # First argument is the CXX compiler
 #CXX_COMPILER=$1
 
-
 # Clone Thrust repository 
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
@@ -17,7 +16,7 @@ git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 # Build Thrust tests
-#cmake --build thrust/build
+cmake --build thrust/build
 
 
 # Run Thrust tests

From 527b34967d041bca70e0e5279c4a0c2f497ac3e2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 19:55:24 -0500
Subject: [PATCH 051/395] Upload entire thrust directory as artifact.

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2dbce674ce6..f31105de601 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -54,7 +54,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: thrust-build-artifacts
-          path: ./thrust/build/
+          path: ./thrust/
   test:
     name: Run tests
     needs: build
@@ -69,6 +69,6 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: thrust-build-artifacts
-          path: build
+          path: thrust
       - name: Run Thrust tests
-        run: pwd && ls && ctest --test-dir build --output-on-failure
\ No newline at end of file
+        run: pwd && ls && ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From 2d21a3332444d7a2aa7a34ef9f8c447797d4f131 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 1 May 2023 20:05:38 -0500
Subject: [PATCH 052/395] Fix build name to use compiler name/version.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f31105de601..7d4f5187bdd 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,7 +13,7 @@ concurrency:
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From 17dd0b1206d9b266c902aa3eb599faa0eef3b801 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:03:18 -0500
Subject: [PATCH 053/395] Factor devcontainer version into env variable.

---
 .github/workflows/test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7d4f5187bdd..882cc535e93 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,5 +1,8 @@
 name: CI (GPU)
 
+env:
+  DEVCONTAINER_VERSION: '23.06'
+
 on:
   push:
     branches:
@@ -42,7 +45,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:${{env.DEVCONTAINTER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - uses: actions/checkout@v3
         with:

From cad8ffb11e933db966c5d2793aa5fb42f90b7d5e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:03:55 -0500
Subject: [PATCH 054/395] Add 'exe' field to compiler yaml definition.

---
 .github/workflows/test.yml | 41 +++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 882cc535e93..73898095830 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,26 +22,27 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-          # gcc
-          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc7'}
-          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc8'}
-          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'gcc9'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc7'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc8'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc9'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'gcc10'}
-          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'gcc11'}
-          - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12'}}
-          # clang
-          #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: 'llvm9'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm9'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm10'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm11'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm12'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm13'}
-          #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: 'llvm14'}
-          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm15'}
-          #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: 'llvm16'}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+        
+        # clang
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
+        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
+
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From e3799373c7189ac21111fab72993c86e7e313bdf Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:04:07 -0500
Subject: [PATCH 055/395] Add workflow step to verify compiler version.

---
 .github/workflows/test.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 73898095830..8f59b68b562 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -48,6 +48,13 @@ jobs:
       options: -u root
       image: rapidsai/devcontainers:${{env.DEVCONTAINTER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
+      - name: Verify compiler version
+        run: |
+          version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+          if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
+            echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+            exit 1
+          fi
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

From dd2098d8299b7c6540133618aada7d48e3fa65de Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:21:27 -0500
Subject: [PATCH 056/395] Only build Thrust CPP/CPP config to make it faster.

---
 ci/build_thrust.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 1afe75d01b6..d6bab2be299 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -13,7 +13,8 @@
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
-cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+cmake -S thrust -B thrust/build -GNinja -DTHRUST_DEVICE_SYSTEM=CPP
 
 # Build Thrust tests
 cmake --build thrust/build

From d643c0d4124ca0554736c00797871df9aa79e6a6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:21:53 -0500
Subject: [PATCH 057/395] tar build artifacts to make upload/download faster.

---
 .github/workflows/test.yml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f59b68b562..8f4178adb14 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -60,12 +60,14 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Build Thrust tests
-        run: bash ./ci/build_thrust.sh
+        run: bash ./ci/build_thrust.sh 
+      - name: tar Thrust build artifacts
+        run: tar czf thrust-build-artifacts.tar.gz ./thrust
       - name: Upload Thrust build artifacts
         uses: actions/upload-artifact@v3
         with:
           name: thrust-build-artifacts
-          path: ./thrust/
+          path: thrust-build-artifacts.tar.gz
   test:
     name: Run tests
     needs: build
@@ -81,5 +83,11 @@ jobs:
         with:
           name: thrust-build-artifacts
           path: thrust
+      - name Untar Thrust build artifacts
+        run: tar xzf thrust-build-artifacts.tar.gz
       - name: Run Thrust tests
-        run: pwd && ls && ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file
+        run: | 
+          pwd 
+          ll thrust/build/bin
+          chmod -R +x thrust/build/bin/*
+          ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From 2a8e614139d1b7973db7ddab4f62b58ba465621a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:22:06 -0500
Subject: [PATCH 058/395] Only enable one matrix job for testing.

---
 .github/workflows/test.yml | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f4178adb14..110d2b8fd15 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -22,26 +22,26 @@ jobs:
       max-parallel: 8
       matrix:
         include:
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
+        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
         
         # clang
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-        - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
+        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
+        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
 
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:

From 10a8967590689ac752cccb69a641f8cd46841ad6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:22:49 -0500
Subject: [PATCH 059/395] Fix yaml syntax.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 110d2b8fd15..0b8d73064eb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -83,7 +83,7 @@ jobs:
         with:
           name: thrust-build-artifacts
           path: thrust
-      - name Untar Thrust build artifacts
+      - name: Untar Thrust build artifacts
         run: tar xzf thrust-build-artifacts.tar.gz
       - name: Run Thrust tests
         run: | 

From 66da2f49174193456906a1c6459bc54bedfa711e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:23:31 -0500
Subject: [PATCH 060/395] typo

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0b8d73064eb..98df091824f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -46,7 +46,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:${{env.DEVCONTAINTER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:${{env.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Verify compiler version
         run: |

From bd1820c5d5ff09f5ab46d1cbf5e2d1a9609dfb16 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:27:45 -0500
Subject: [PATCH 061/395] Fix devcontainer version env by putting it in
 defaults.

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 98df091824f..c92d393bb0c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,7 +1,8 @@
 name: CI (GPU)
 
-env:
-  DEVCONTAINER_VERSION: '23.06'
+defaults:
+ env:
+   DEVCONTAINER_VERSION: '23.06'
 
 on:
   push:

From 46877597f74ff7fa88e809bb3de610cc1182b125 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:31:09 -0500
Subject: [PATCH 062/395] Remove env. from DEVCONAINER_VERSION.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c92d393bb0c..7c303ce5d21 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -47,7 +47,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:${{env.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:${{DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Verify compiler version
         run: |

From 542317a388c54345cdfa78eeef8699ae3148add8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:32:35 -0500
Subject: [PATCH 063/395] Don't use defaults?

---
 .github/workflows/test.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7c303ce5d21..f32a9197787 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,8 +1,7 @@
 name: CI (GPU)
 
-defaults:
- env:
-   DEVCONTAINER_VERSION: '23.06'
+env:
+  DEVCONTAINER_VERSION: '23.06'
 
 on:
   push:

From 0357a5759aa7f69c1203764e09c1fb268653df69 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:33:34 -0500
Subject: [PATCH 064/395] Add back env.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f32a9197787..98df091824f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -46,7 +46,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:${{DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:${{env.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Verify compiler version
         run: |

From a0953e4a4c8b7cf18815a0d9e02c992106f4b4f5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:35:41 -0500
Subject: [PATCH 065/395] Just hardcode the devcontainer version.

---
 .github/workflows/test.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 98df091824f..f50331c7088 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,8 +1,5 @@
 name: CI (GPU)
 
-env:
-  DEVCONTAINER_VERSION: '23.06'
-
 on:
   push:
     branches:
@@ -46,7 +43,7 @@ jobs:
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devcontainers:${{env.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
       - name: Verify compiler version
         run: |

From 2298071f5ba3a70c064800297869918f30ce4850 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:43:18 -0500
Subject: [PATCH 066/395] Update compiler version check to use bash shell.

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f50331c7088..f510b6a3cd8 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -41,6 +41,7 @@ jobs:
         #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
 
     runs-on: linux-${{ matrix.cpu }}-cpu16
+    shell: bash
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
@@ -52,6 +53,7 @@ jobs:
             echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
             exit 1
           fi
+          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

From 0b459cd0c5ef772faedce6aeb6589ee338ee589d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:49:23 -0500
Subject: [PATCH 067/395] Print size of artifact tarball.

---
 .github/workflows/test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f510b6a3cd8..60495762266 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -61,7 +61,9 @@ jobs:
       - name: Build Thrust tests
         run: bash ./ci/build_thrust.sh 
       - name: tar Thrust build artifacts
-        run: tar czf thrust-build-artifacts.tar.gz ./thrust
+        run: |
+          tar czf thrust-build-artifacts.tar.gz ./thrust
+          echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
       - name: Upload Thrust build artifacts
         uses: actions/upload-artifact@v3
         with:

From a2a403f072cf411f04d94a390b00122b7225ea0a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:51:49 -0500
Subject: [PATCH 068/395] Don't download artifact into thrust/.

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 60495762266..c539fcb8d07 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -83,7 +83,6 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: thrust-build-artifacts
-          path: thrust
       - name: Untar Thrust build artifacts
         run: tar xzf thrust-build-artifacts.tar.gz
       - name: Run Thrust tests

From 9ceb6aff7a0da91a61f4bbc250e70656d3a6830b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 10:54:33 -0500
Subject: [PATCH 069/395] Set shell as global default.

---
 .github/workflows/test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index c539fcb8d07..23ec3161e48 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,5 +1,9 @@
 name: CI (GPU)
 
+defaults:
+  run:
+    shell: bash
+
 on:
   push:
     branches:
@@ -41,7 +45,6 @@ jobs:
         #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
 
     runs-on: linux-${{ matrix.cpu }}-cpu16
-    shell: bash
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}

From e7c326fb4a0b75d65d048b6f5b629bb51580bf6b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 11:03:08 -0500
Subject: [PATCH 070/395] Skip compiler version verification for now.

---
 .github/workflows/test.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 23ec3161e48..0a1e7fe6525 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -49,14 +49,14 @@ jobs:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     steps:
-      - name: Verify compiler version
-        run: |
-          version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-          if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
-            echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
-            exit 1
-          fi
-          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
+#      - name: Verify compiler version
+#        run: |
+#          version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+#          if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
+#            echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+#            exit 1
+#          fi
+#          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

From 8f4bbe2ce08f5b2183fd09fe52d6c7590792b7d0 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 11:39:05 -0500
Subject: [PATCH 071/395] Don't use ll.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0a1e7fe6525..f9de8005124 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -91,6 +91,6 @@ jobs:
       - name: Run Thrust tests
         run: | 
           pwd 
-          ll thrust/build/bin
+          ls
           chmod -R +x thrust/build/bin/*
           ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From 54f30e69530723f6afe25314db06afb6cdded1f2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 12:00:54 -0500
Subject: [PATCH 072/395] Add step to remove unused object files.

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f9de8005124..33d3132cd5c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -63,6 +63,8 @@ jobs:
           persist-credentials: false
       - name: Build Thrust tests
         run: bash ./ci/build_thrust.sh 
+      - name: Clean up object files
+        run: rm -rf ./thrust/build/testing
       - name: tar Thrust build artifacts
         run: |
           tar czf thrust-build-artifacts.tar.gz ./thrust

From 8f5ea60bed72661e51ae3660955a785bbfffc5c4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 12:40:42 -0500
Subject: [PATCH 073/395] Add fetching AWS credentials.

---
 .github/workflows/test.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 33d3132cd5c..7d2f676b306 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -48,6 +48,8 @@ jobs:
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write
     steps:
 #      - name: Verify compiler version
 #        run: |
@@ -57,6 +59,12 @@ jobs:
 #            exit 1
 #          fi
 #          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
+      - name: Get AWS credentials for sccache bucket
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nvidia
+          aws-region: us-east-2 
+          role-duration-seconds: 43200 # 12 hours
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

From 43eef303544b9b38d99bc56e29eefbf5a14bf6b9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 12:48:54 -0500
Subject: [PATCH 074/395] Add sccache env variables and add step to test
 sccache.

---
 .github/workflows/test.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7d2f676b306..a62ff730e0c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,5 +1,12 @@
 name: CI (GPU)
 
+env:
+  SCCACHE_BUCKET: rapids-sccache-east
+  SCCACHE_REGION: us-east-2
+  SCCACHE_IDLE_TIMEOUT: 32768
+  SCCACHE_S3_USE_SSL: true
+  SCCACHE_S3_NO_CREDENTIALS: false
+
 defaults:
   run:
     shell: bash
@@ -65,6 +72,8 @@ jobs:
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nvidia
           aws-region: us-east-2 
           role-duration-seconds: 43200 # 12 hours
+      - name: Test sccache
+        run: sccache -s
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

From 5806b87ed455faa78ae3e6bc990ab56847b94611 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 12:50:48 -0500
Subject: [PATCH 075/395] Use v2 of configure aws credentials action.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a62ff730e0c..5f13c3adfa5 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -67,7 +67,7 @@ jobs:
 #          fi
 #          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
       - name: Get AWS credentials for sccache bucket
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nvidia
           aws-region: us-east-2 

From aea97387b297e89fc3f5f629d26ca3856c7aae56 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:00:30 -0500
Subject: [PATCH 076/395] Move sccache env variables to Thrust build step.

---
 .github/workflows/test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5f13c3adfa5..6f10a2e5420 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,11 +1,5 @@
 name: CI (GPU)
 
-env:
-  SCCACHE_BUCKET: rapids-sccache-east
-  SCCACHE_REGION: us-east-2
-  SCCACHE_IDLE_TIMEOUT: 32768
-  SCCACHE_S3_USE_SSL: true
-  SCCACHE_S3_NO_CREDENTIALS: false
 
 defaults:
   run:
@@ -79,6 +73,12 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Build Thrust tests
+        env:
+          SCCACHE_BUCKET: rapids-sccache-east
+          SCCACHE_REGION: us-east-2
+          SCCACHE_IDLE_TIMEOUT: 32768
+          SCCACHE_S3_USE_SSL: true
+          SCCACHE_S3_NO_CREDENTIALS: false
         run: bash ./ci/build_thrust.sh 
       - name: Clean up object files
         run: rm -rf ./thrust/build/testing

From 34d36d49f7454121a262bde1343320bfd61bee1b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:02:51 -0500
Subject: [PATCH 077/395] Test sccache stats before/after build.

---
 .github/workflows/test.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6f10a2e5420..bf062283f03 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -66,8 +66,6 @@ jobs:
           role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nvidia
           aws-region: us-east-2 
           role-duration-seconds: 43200 # 12 hours
-      - name: Test sccache
-        run: sccache -s
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -79,7 +77,10 @@ jobs:
           SCCACHE_IDLE_TIMEOUT: 32768
           SCCACHE_S3_USE_SSL: true
           SCCACHE_S3_NO_CREDENTIALS: false
-        run: bash ./ci/build_thrust.sh 
+        run: |
+          sccache -s
+          bash ./ci/build_thrust.sh 
+          sccache -s
       - name: Clean up object files
         run: rm -rf ./thrust/build/testing
       - name: tar Thrust build artifacts

From 3ea58542d4f3fcf0c04913c8570889d3e543090c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:22:11 -0500
Subject: [PATCH 078/395] Move permissions to workflow level.

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index d6bab2be299..8657feba1b6 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -14,7 +14,7 @@ git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
 #cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-cmake -S thrust -B thrust/build -GNinja -DTHRUST_DEVICE_SYSTEM=CPP
+
 
 # Build Thrust tests
 cmake --build thrust/build

From e086bd1d120dab1869ccc49f53fdea7c99898b3c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:22:30 -0500
Subject: [PATCH 079/395] Move permissions to global.

---
 .github/workflows/test.yml | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bf062283f03..78379356a21 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,6 +15,21 @@ concurrency:
   group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
 
+permissions:
+  actions: none
+  checks: none
+  contents: read
+  deployments: none
+  discussions: none
+  id-token: write
+  issues: none
+  packages: read
+  pages: none
+  pull-requests: read
+  repository-projects: none
+  security-events: none
+  statuses: none
+
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
@@ -49,8 +64,6 @@ jobs:
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write
     steps:
 #      - name: Verify compiler version
 #        run: |
@@ -81,8 +94,6 @@ jobs:
           sccache -s
           bash ./ci/build_thrust.sh 
           sccache -s
-      - name: Clean up object files
-        run: rm -rf ./thrust/build/testing
       - name: tar Thrust build artifacts
         run: |
           tar czf thrust-build-artifacts.tar.gz ./thrust

From b0bf0921ec166fa125069a84fc0c757b056de013 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:37:36 -0500
Subject: [PATCH 080/395] Actually enable building in the build script.

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 8657feba1b6..042715ab178 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -13,7 +13,7 @@
 git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 
 # Build Thrust tests

From 90d70d1f416eedfe503c0d988c2aadc4f273e9c6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 13:44:36 -0500
Subject: [PATCH 081/395] Update AWS role.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 78379356a21..223d68e5224 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -76,7 +76,7 @@ jobs:
       - name: Get AWS credentials for sccache bucket
         uses: aws-actions/configure-aws-credentials@v2
         with:
-          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nvidia
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
           aws-region: us-east-2 
           role-duration-seconds: 43200 # 12 hours
       - uses: actions/checkout@v3

From 8a9609ab14efd0b7baa22d4ed73596793caa38ac Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 14:30:10 -0500
Subject: [PATCH 082/395] Scope permissions to the build job.

---
 .github/workflows/test.yml | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 223d68e5224..90a61db5e44 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,21 +15,6 @@ concurrency:
   group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
 
-permissions:
-  actions: none
-  checks: none
-  contents: read
-  deployments: none
-  discussions: none
-  id-token: write
-  issues: none
-  packages: read
-  pages: none
-  pull-requests: read
-  repository-projects: none
-  security-events: none
-  statuses: none
-
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
@@ -64,6 +49,9 @@ jobs:
     container:
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write 
+      contents: read 
     steps:
 #      - name: Verify compiler version
 #        run: |

From 575bde3cd789733354e37e7e632ae2e36eba900b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 15:04:43 -0500
Subject: [PATCH 083/395] Add timing to the build step.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 90a61db5e44..f64f0851312 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -80,7 +80,7 @@ jobs:
           SCCACHE_S3_NO_CREDENTIALS: false
         run: |
           sccache -s
-          bash ./ci/build_thrust.sh 
+          time bash ./ci/build_thrust.sh 
           sccache -s
       - name: tar Thrust build artifacts
         run: |

From a4cd9ada8dfbd28a74e681d85e1b1f865e8aa257 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 2 May 2023 16:32:12 -0500
Subject: [PATCH 084/395] Remove sccache stats before build as it isn't useful.

---
 .github/workflows/test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f64f0851312..9c448cfcf72 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -79,7 +79,6 @@ jobs:
           SCCACHE_S3_USE_SSL: true
           SCCACHE_S3_NO_CREDENTIALS: false
         run: |
-          sccache -s
           time bash ./ci/build_thrust.sh 
           sccache -s
       - name: tar Thrust build artifacts

From 13b04fd9529f31e1e15c86a1b24387a3b7d3cddd Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 3 May 2023 10:31:54 -0500
Subject: [PATCH 085/395] Move matrix to file.

---
 .github/workflows/test.yml | 200 ++++++++++++++++++++-----------------
 ci/matrix.yaml             |  21 ++++
 2 files changed, 127 insertions(+), 94 deletions(-)
 create mode 100644 ci/matrix.yaml

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9c448cfcf72..0bdec73a3ec 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,99 +16,111 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix:
-        include:
-        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
-        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
-        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
-        
-        # clang
-        #- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-        #- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
-
-    runs-on: linux-${{ matrix.cpu }}-cpu16
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write 
-      contents: read 
+  compute-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
     steps:
-#      - name: Verify compiler version
-#        run: |
-#          version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-#          if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
-#            echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
-#            exit 1
-#          fi
-#          echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
-      - name: Get AWS credentials for sccache bucket
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-          aws-region: us-east-2 
-          role-duration-seconds: 43200 # 12 hours
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Build Thrust tests
-        env:
-          SCCACHE_BUCKET: rapids-sccache-east
-          SCCACHE_REGION: us-east-2
-          SCCACHE_IDLE_TIMEOUT: 32768
-          SCCACHE_S3_USE_SSL: true
-          SCCACHE_S3_NO_CREDENTIALS: false
-        run: |
-          time bash ./ci/build_thrust.sh 
-          sccache -s
-      - name: tar Thrust build artifacts
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
         run: |
-          tar czf thrust-build-artifacts.tar.gz ./thrust
-          echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
-      - name: Upload Thrust build artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: thrust-build-artifacts
-          path: thrust-build-artifacts.tar.gz
-  test:
-    name: Run tests
-    needs: build
-    runs-on: linux-amd64-gpu-v100-latest-1
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
-      env:
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-    steps:
-      - name: Download Thrust build artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: thrust-build-artifacts
-      - name: Untar Thrust build artifacts
-        run: tar xzf thrust-build-artifacts.tar.gz
-      - name: Run Thrust tests
-        run: | 
-          pwd 
-          ls
-          chmod -R +x thrust/build/bin/*
-          ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file
+          MATRIX=$(yq -r . ./ci/matrix.yaml | jq -c '.[]')
+          echo "${MATRIX}"
+
+  #build:
+  #  if: github.repository #== 'NVIDIA/cccl'
+  #  name: Build (CUDA ${{m#atrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+  #  strategy:
+  #    fail-fast: false
+  #    max-parallel: 8
+  #    matrix:
+  #      include:
+  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
+  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+  #      
+  #       clang
+  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
+  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
+  #  runs-on: linux-${{ mat#rix.cpu }}-cpu16
+  #  container:
+  #    options: -u root
+  #    image: rapidsai/devc#ontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+  #  permissions:
+  #    id-token: write 
+  #    contents: read 
+  #  steps:
+# #     - name: Verify comp#iler version
+# #       run: |
+# #         version=$("${{m#atrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+# #         if [[ "${versio#n}" != "${{ matrix.compiler.version }}" ]]; then
+# #           echo "Compile#r version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+# #           exit 1
+# #         fi
+# #         echo "Sucessful#ly found ${{matrix.compiler.name}} ${version}"
+  #    - name: Get AWS cred#entials for sccache bucket
+  #      uses: aws-actions/#configure-aws-credentials@v2
+  #      with:
+  #        role-to-assume: #arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+  #        aws-region: us-e#ast-2 
+  #        role-duration-se#conds: 43200 # 12 hours
+  #    - uses: actions/chec#kout@v3
+  #      with:
+  #        fetch-depth: 0
+  #        persist-credenti#als: false
+  #    - name: Build Thrust# tests
+  #      env:
+  #        SCCACHE_BUCKET: #rapids-sccache-east
+  #        SCCACHE_REGION: #us-east-2
+  #        SCCACHE_IDLE_TIM#EOUT: 32768
+  #        SCCACHE_S3_USE_S#SL: true
+  #        SCCACHE_S3_NO_CR#EDENTIALS: false
+  #      run: |
+  #        time bash ./ci/b#uild_thrust.sh 
+  #        sccache -s
+  #    - name: tar Thrust b#uild artifacts
+  #      run: |
+  #        tar czf thrust-b#uild-artifacts.tar.gz ./thrust
+  #        echo "tarball si#ze: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
+  #    - name: Upload Thrus#t build artifacts
+  #      uses: actions/uplo#ad-artifact@v3
+  #      with:
+  #        name: thrust-bui#ld-artifacts
+  #        path: thrust-bui#ld-artifacts.tar.gz
+  #test:
+  #  name: Run tests
+  #  needs: build
+  #  runs-on: linux-amd64-g#pu-v100-latest-1
+  #  container:
+  #    options: -u root
+  #    image: rapidsai/devc#ontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
+  #    env:
+  #      NVIDIA_VISIBLE_DEV#ICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+  #  steps:
+  #    - name: Download Thr#ust build artifacts
+  #      uses: actions/down#load-artifact@v3
+  #      with:
+  #        name: thrust-bui#ld-artifacts
+  #    - name: Untar Thrust# build artifacts
+  #      run: tar xzf thrus#t-build-artifacts.tar.gz
+  #    - name: Run Thrust t#ests
+  #      run: | 
+  #        pwd 
+  #        ls
+  #        chmod -R +x thru#st/build/bin/*
+  #        ctest --test-dir# thrust/build --output-on-failure
\ No newline at end of file
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
new file mode 100644
index 00000000000..065bbe88ab6
--- /dev/null
+++ b/ci/matrix.yaml
@@ -0,0 +1,21 @@
+#gcc
+- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
+- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+
+ #clang
+- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
+- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
\ No newline at end of file

From 15e4c75432c1252910ee588b6827353ecb819122 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 3 May 2023 10:33:25 -0500
Subject: [PATCH 086/395] Fix errant #

---
 ci/matrix.yaml | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 065bbe88ab6..b57a522fbac 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -1,21 +1,21 @@
 #gcc
-- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
-- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
 
  #clang
-- {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-- {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
\ No newline at end of file
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
\ No newline at end of file

From 7e4d00143ef10f1711c03f1eb16e43020e757051 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 3 May 2023 10:34:19 -0500
Subject: [PATCH 087/395] Cat matrix.yaml.

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0bdec73a3ec..404fadbcc28 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,6 +26,7 @@ jobs:
       - name: Compute matrix
         id: compute-matrix
         run: |
+          cat ./ci/matrix.yaml
           MATRIX=$(yq -r . ./ci/matrix.yaml | jq -c '.[]')
           echo "${MATRIX}"
 

From c0d52c8eb85928a1dce4158fdeb5357becc49e80 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 11:54:12 -0500
Subject: [PATCH 088/395] Fix the yaml parsing command and output to
 GITHUB_OUTPUT.

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 404fadbcc28..44d21832d4b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,8 +27,8 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -r . ./ci/matrix.yaml | jq -c '.[]')
-          echo "${MATRIX}"
+          MATRIX=$(yq -o=json ./ci/matrix.yaml)
+          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   #build:
   #  if: github.repository #== 'NVIDIA/cccl'

From c54287bb5da81c70ad286f492c66a8a6decf2084 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 11:56:25 -0500
Subject: [PATCH 089/395] Need jq after all.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 44d21832d4b..2c805bf3752 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml)
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.[]')
           echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   #build:

From aae8bf8cb5d30f32e4756ff11e0aa49257fd940b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:06:35 -0500
Subject: [PATCH 090/395] Use printf.

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2c805bf3752..f6eb5b5d81d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,8 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.[]')
-          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
+          printf -v MATRIX_STRING 'MATRIX=%q' "${MATRIX}"
+          echo "${MATRIX_STRING}" | tee -a "${GITHUB_OUTPUT}"
 
   #build:
   #  if: github.repository #== 'NVIDIA/cccl'

From b5bfb077334fc7b736841f5e996f72602305136c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:09:46 -0500
Subject: [PATCH 091/395] Use jq -c.

---
 .github/workflows/test.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index f6eb5b5d81d..a0ddc7c609f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,9 +27,8 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.[]')
-          printf -v MATRIX_STRING 'MATRIX=%q' "${MATRIX}"
-          echo "${MATRIX_STRING}" | tee -a "${GITHUB_OUTPUT}"
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
+          echo "MATRIX=${MATRIX}" >> ${GITHUB_OUTPUT}
 
   #build:
   #  if: github.repository #== 'NVIDIA/cccl'

From b82b32feac00664fea6bca94f6a4130fcd2aca16 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:11:02 -0500
Subject: [PATCH 092/395] print the matrix.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0ddc7c609f..59aebb55f4d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          echo "MATRIX=${MATRIX}" >> ${GITHUB_OUTPUT}
+          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   #build:
   #  if: github.repository #== 'NVIDIA/cccl'

From f8b08046aa47745c816f24f77b1cfe39d11ece8f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:32:05 -0500
Subject: [PATCH 093/395] Test that the matrix can be used in subsequent job.

---
 .github/workflows/test.yml | 52 +++++++++++++-------------------------
 1 file changed, 17 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 59aebb55f4d..ff8a97079bc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,41 +30,23 @@ jobs:
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
           echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
-  #build:
-  #  if: github.repository #== 'NVIDIA/cccl'
-  #  name: Build (CUDA ${{m#atrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-  #  strategy:
-  #    fail-fast: false
-  #    max-parallel: 8
-  #    matrix:
-  #      include:
-  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
-  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
-  #      
-  #       clang
-  #      - {cuda: '11.0', u#buntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-  #      - {cuda: '12.1', u#buntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
-  #  runs-on: linux-${{ mat#rix.cpu }}-cpu16
-  #  container:
-  #    options: -u root
-  #    image: rapidsai/devc#ontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-  #  permissions:
-  #    id-token: write 
-  #    contents: read 
+  build:
+    if: github.repository == 'NVIDIA/cccl'
+    name: Build (CUDA ${{m#atrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: linux-${{ mat#rix.cpu }}-cpu16
+    container:
+      options: -u root
+      image: rapidsai/devc#ontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write 
+      contents: read 
+    steps:
+      - name: test
+        run: echo "hello"
   #  steps:
 # #     - name: Verify comp#iler version
 # #       run: |

From 55a1ec3ab7a65afa9904cf756b2f6575c8d2d3f5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:32:42 -0500
Subject: [PATCH 094/395] fix errant #

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ff8a97079bc..6aefe4953c4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,7 @@ jobs:
 
   build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{m#atrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From ca48fc49ba55dfe23374bb8ef575893d9653cf12 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:34:36 -0500
Subject: [PATCH 095/395] Fix all errant #.

---
 .github/workflows/test.yml | 78 +++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6aefe4953c4..bd08eddc350 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -37,10 +37,10 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: linux-${{ mat#rix.cpu }}-cpu16
+    runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root
-      image: rapidsai/devc#ontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
     permissions:
       id-token: write 
       contents: read 
@@ -48,62 +48,62 @@ jobs:
       - name: test
         run: echo "hello"
   #  steps:
-# #     - name: Verify comp#iler version
+# #     - name: Verify compiler version
 # #       run: |
-# #         version=$("${{m#atrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-# #         if [[ "${versio#n}" != "${{ matrix.compiler.version }}" ]]; then
-# #           echo "Compile#r version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+# #         version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+# #         if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
+# #           echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
 # #           exit 1
 # #         fi
-# #         echo "Sucessful#ly found ${{matrix.compiler.name}} ${version}"
-  #    - name: Get AWS cred#entials for sccache bucket
-  #      uses: aws-actions/#configure-aws-credentials@v2
+# #         echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
+  #    - name: Get AWS credentials for sccache bucket
+  #      uses: aws-actions/configure-aws-credentials@v2
   #      with:
-  #        role-to-assume: #arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-  #        aws-region: us-e#ast-2 
-  #        role-duration-se#conds: 43200 # 12 hours
-  #    - uses: actions/chec#kout@v3
+  #        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+  #        aws-region: us-east-2 
+  #        role-duration-seconds: 43200 # 12 hours
+  #    - uses: actions/checkout@v3
   #      with:
   #        fetch-depth: 0
-  #        persist-credenti#als: false
-  #    - name: Build Thrust# tests
+  #        persist-credentials: false
+  #    - name: Build Thrust tests
   #      env:
-  #        SCCACHE_BUCKET: #rapids-sccache-east
-  #        SCCACHE_REGION: #us-east-2
-  #        SCCACHE_IDLE_TIM#EOUT: 32768
-  #        SCCACHE_S3_USE_S#SL: true
-  #        SCCACHE_S3_NO_CR#EDENTIALS: false
+  #        SCCACHE_BUCKET: rapids-sccache-east
+  #        SCCACHE_REGION: us-east-2
+  #        SCCACHE_IDLE_TIMEOUT: 32768
+  #        SCCACHE_S3_USE_SSL: true
+  #        SCCACHE_S3_NO_CREDENTIALS: false
   #      run: |
-  #        time bash ./ci/b#uild_thrust.sh 
+  #        time bash ./ci/build_thrust.sh 
   #        sccache -s
-  #    - name: tar Thrust b#uild artifacts
+  #    - name: tar Thrust build artifacts
   #      run: |
-  #        tar czf thrust-b#uild-artifacts.tar.gz ./thrust
-  #        echo "tarball si#ze: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
-  #    - name: Upload Thrus#t build artifacts
-  #      uses: actions/uplo#ad-artifact@v3
+  #        tar czf thrust-build-artifacts.tar.gz ./thrust
+  #        echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
+  #    - name: Upload Thrust build artifacts
+  #      uses: actions/upload-artifact@v3
   #      with:
-  #        name: thrust-bui#ld-artifacts
-  #        path: thrust-bui#ld-artifacts.tar.gz
+  #        name: thrust-build-artifacts
+  #        path: thrust-build-artifacts.tar.gz
   #test:
   #  name: Run tests
   #  needs: build
-  #  runs-on: linux-amd64-g#pu-v100-latest-1
+  #  runs-on: linux-amd64-gpu-v100-latest-1
   #  container:
   #    options: -u root
-  #    image: rapidsai/devc#ontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
+  #    image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
   #    env:
-  #      NVIDIA_VISIBLE_DEV#ICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+  #      NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
   #  steps:
-  #    - name: Download Thr#ust build artifacts
-  #      uses: actions/down#load-artifact@v3
+  #    - name: Download Thrust build artifacts
+  #      uses: actions/download-artifact@v3
   #      with:
-  #        name: thrust-bui#ld-artifacts
-  #    - name: Untar Thrust# build artifacts
-  #      run: tar xzf thrus#t-build-artifacts.tar.gz
-  #    - name: Run Thrust t#ests
+  #        name: thrust-build-artifacts
+  #    - name: Untar Thrust build artifacts
+  #      run: tar xzf thrust-build-artifacts.tar.gz
+  #    - name: Run Thrust tests
   #      run: | 
   #        pwd 
   #        ls
-  #        chmod -R +x thru#st/build/bin/*
-  #        ctest --test-dir# thrust/build --output-on-failure
\ No newline at end of file
+  #        chmod -R +x thrust/build/bin/*
+  #        ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From 0a15b02cdee4ee7fd9e5d2405eecafcf02a8448a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:39:40 -0500
Subject: [PATCH 096/395] Use matrix directly without fromJSON

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bd08eddc350..42d971f0246 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From 461fc02eff304884d75b043f5514830c5083dba6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 12:49:09 -0500
Subject: [PATCH 097/395] Surround matrix with quotes?

---
 .github/workflows/test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 42d971f0246..5b698b8967c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,8 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
+          \"$MATRIX\
+          echo "MATRIX=\"${MATRIX}\"" | tee -a "$GITHUB_OUTPUT"
 
   build:
     if: github.repository == 'NVIDIA/cccl'
@@ -36,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From ce7911440789bade1a0af75a8ba0dbc967c2060b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 13:02:30 -0500
Subject: [PATCH 098/395] Use matrix include:

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5b698b8967c..85f9ee946f6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,8 +28,7 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          \"$MATRIX\
-          echo "MATRIX=\"${MATRIX}\"" | tee -a "$GITHUB_OUTPUT"
+          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   build:
     if: github.repository == 'NVIDIA/cccl'
@@ -37,7 +36,8 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      matrix: 
+        include: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From 9fffc832b01d30daabccfd6bd74ba6bdf6fbc196 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 13:15:55 -0500
Subject: [PATCH 099/395] quotes

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 85f9ee946f6..a0cbe6906ed 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -37,7 +37,7 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix: 
-        include: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+        include: ${{ fromJSON('needs.compute-matrix.outputs.MATRIX') }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From ae9aa029a4dfd6a57a8cc89b56afb0aba6ea6cb9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 13:27:42 -0500
Subject: [PATCH 100/395] Prepend with include:

---
 .github/workflows/test.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0cbe6906ed..2a683138b65 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          echo "MATRIX=${MATRIX}" | tee -a "$GITHUB_OUTPUT"
+          echo "MATRIX={\"include\":$MATRIX}"" | tee -a "$GITHUB_OUTPUT"
 
   build:
     if: github.repository == 'NVIDIA/cccl'
@@ -36,8 +36,7 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: 
-        include: ${{ fromJSON('needs.compute-matrix.outputs.MATRIX') }}
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container:
       options: -u root

From 5a2420a15778bda4dade9a43af7fac231a317eba Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 13:28:14 -0500
Subject: [PATCH 101/395] Too many quotes.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2a683138b65..8cb06516523 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
         run: |
           cat ./ci/matrix.yaml
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          echo "MATRIX={\"include\":$MATRIX}"" | tee -a "$GITHUB_OUTPUT"
+          echo "MATRIX={\"include\":$MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   build:
     if: github.repository == 'NVIDIA/cccl'

From 1d246dd9cc989665f3fe5fc0b7228deae028a851 Mon Sep 17 00:00:00 2001
From: AJ Schmidt <aschmidt@nvidia.com>
Date: Thu, 4 May 2023 14:54:17 -0400
Subject: [PATCH 102/395] use `jq` to construct final `json` object

---
 .github/workflows/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8cb06516523..dec9d9d628e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,8 +27,8 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c)
-          echo "MATRIX={\"include\":$MATRIX}" | tee -a "$GITHUB_OUTPUT"
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '{include: .}')
+          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   build:
     if: github.repository == 'NVIDIA/cccl'

From 970b8dfa6f782b5dfc4d0bb82278ae78c0b43d97 Mon Sep 17 00:00:00 2001
From: AJ Schmidt <aschmidt@nvidia.com>
Date: Thu, 4 May 2023 14:54:55 -0400
Subject: [PATCH 103/395] add final newline to `matrix.yaml` for log
 readability

---
 ci/matrix.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index b57a522fbac..f2661a84445 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -18,4 +18,4 @@
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
 - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
\ No newline at end of file
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}

From d30d0b3e6dabdb5c6fd77c9c941247716cdfa56f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 14:02:38 -0500
Subject: [PATCH 104/395] build job needs to depend on compute-matrix.

---
 .github/workflows/test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8cb06516523..95aa96e1b8d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,6 +31,7 @@ jobs:
           echo "MATRIX={\"include\":$MATRIX}" | tee -a "$GITHUB_OUTPUT"
 
   build:
+    needs: compute-matrix
     if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:

From 51f4adcdef3d2b66f9035885c49d025d4d229658 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 14:49:26 -0500
Subject: [PATCH 105/395] Move the include: to matrix file and add std.

---
 .github/workflows/test.yml | 2 +-
 ci/matrix.yaml             | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 21bb730f2d6..9d76044127c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '{include: .}')
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.')
           echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   build:
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index f2661a84445..b68b23c6649 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -1,4 +1,8 @@
 #gcc
+
+std: [11, 14, 17, 20]
+
+include:
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}

From eead1c8233747ef944f8333420bca39acadd6e49 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 14:52:27 -0500
Subject: [PATCH 106/395] Add std to job name.

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9d76044127c..637e3457de1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,7 +33,7 @@ jobs:
   build:
     needs: compute-matrix
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From 73f3f327f8ca79f9a0e2ea2451b920e37a01c76d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:06:53 -0500
Subject: [PATCH 107/395] test manual matrix

---
 .github/workflows/test.yml | 70 +++++++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 637e3457de1..855cf33c785 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,38 +16,54 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  compute-matrix:
+  test-matrix:
     runs-on: ubuntu-latest
-    outputs:
-      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Compute matrix
-        id: compute-matrix
-        run: |
-          cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.')
-          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
-
-  build:
-    needs: compute-matrix
-    if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: linux-${{ matrix.cpu }}-cpu16
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write 
-      contents: read 
-    steps:
-      - name: test
-        run: echo "hello"
+      matrix: 
+        std: [11, 14, 17, 20]
+        include:
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+
+  #compute-matrix:
+  #  runs-on: ubuntu-latest
+  #  outputs:
+  #    MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
+  #  steps:
+  #    - name: Checkout repo
+  #      uses: actions/checkout@v3
+  #    - name: Compute matrix
+  #      id: compute-matrix
+  #      run: |
+  #        cat ./ci/matrix.yaml
+  #        MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.')
+  #        echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
+
+  #build:
+  #  needs: compute-matrix
+  #  if: github.repository == 'NVIDIA/cccl'
+  #  name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+  #  strategy:
+  #    fail-fast: false
+  #    max-parallel: 8
+  #    matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+  #  runs-on: linux-${{ matrix.cpu }}-cpu16
+  #  container:
+  #    options: -u root
+  #    image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+  #  permissions:
+  #    id-token: write 
+  #    contents: read 
+  #  steps:
+  #    - name: test
+  #      run: echo "hello"
   #  steps:
 # #     - name: Verify compiler version
 # #       run: |

From 5fe6355399b96c89f5364a171aa46bf0c2ed8b4e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:07:43 -0500
Subject: [PATCH 108/395] add run

---
 .github/workflows/test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 855cf33c785..083d5e28c78 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -31,6 +31,8 @@ jobs:
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+    steps:
+      -run: echo "hello"
 
   #compute-matrix:
   #  runs-on: ubuntu-latest

From aac8ff16c9dca66585e2d6f444b990e034780219 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:08:38 -0500
Subject: [PATCH 109/395] fix

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 083d5e28c78..efeba8fcb2f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,7 +32,8 @@ jobs:
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
         - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
     steps:
-      -run: echo "hello"
+      - name: test
+        run: echo "hello"
 
   #compute-matrix:
   #  runs-on: ubuntu-latest

From 459c51897c9671f488c32c0dba3c7fef9751b677 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:10:14 -0500
Subject: [PATCH 110/395] Add another matrix axis.

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index efeba8fcb2f..a2a335476d4 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -23,7 +23,8 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix: 
-        std: [11, 14, 17, 20]
+        lib: [thrust, cub, libcudacxx]
+        std: ['11', '14', '17', '20']
         include:
         - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
         - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}

From db5b431a0c40f840dbbf1dd6f68008c60785f15c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:13:35 -0500
Subject: [PATCH 111/395] Try a hack.

---
 .github/workflows/test.yml | 12 ++++++------
 ci/thrust                  |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)
 create mode 160000 ci/thrust

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a2a335476d4..0ec4175ae76 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,12 +26,12 @@ jobs:
         lib: [thrust, cub, libcudacxx]
         std: ['11', '14', '17', '20']
         include:
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, unique: 0}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, unique: 1}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, unique: 2}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, unique: 3}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, unique: 4}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, unique: 5}
     steps:
       - name: test
         run: echo "hello"
diff --git a/ci/thrust b/ci/thrust
new file mode 160000
index 00000000000..02931a309be
--- /dev/null
+++ b/ci/thrust
@@ -0,0 +1 @@
+Subproject commit 02931a309bee769853088b79b4e3ab1c0bd2336c

From b577b2352256f2969546022ed73874d47d5069d2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 4 May 2023 15:16:22 -0500
Subject: [PATCH 112/395] Different hack

---
 .github/workflows/test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0ec4175ae76..0cf7216dc90 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -26,12 +26,12 @@ jobs:
         lib: [thrust, cub, libcudacxx]
         std: ['11', '14', '17', '20']
         include:
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, unique: 0}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, unique: 1}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, unique: 2}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, unique: 3}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, unique: 4}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, unique: 5}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, one: 0}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, two: 1}
+        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, three: 2}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, four: 3}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, five: 4}
+        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, six: 5}
     steps:
       - name: test
         run: echo "hello"

From dc8237e10804883a5d204326c18fed6a299e1a5c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 10 May 2023 10:42:12 -0500
Subject: [PATCH 113/395] Update matrix to add C++ dialect as tensor mode.

---
 .github/workflows/test.yml | 144 ++++++++++++++++---------------------
 ci/matrix.yaml             |   6 +-
 2 files changed, 67 insertions(+), 83 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0cf7216dc90..39b61e2fc36 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -16,96 +16,76 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test-matrix:
+  compute-matrix:
     runs-on: ubuntu-latest
+    outputs:
+      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          cat ./ci/matrix.yaml
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ]')
+          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
+
+  build:
+    needs: compute-matrix
+    if: github.repository == 'NVIDIA/cccl'
     name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: 
-        lib: [thrust, cub, libcudacxx]
-        std: ['11', '14', '17', '20']
-        include:
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, one: 0}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, two: 1}
-        - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, three: 2}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, four: 3}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, five: 4}
-        - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, six: 5}
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: linux-${{ matrix.cpu }}-cpu4
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write 
+      contents: read 
     steps:
       - name: test
         run: echo "hello"
-
-  #compute-matrix:
-  #  runs-on: ubuntu-latest
-  #  outputs:
-  #    MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
-  #  steps:
-  #    - name: Checkout repo
-  #      uses: actions/checkout@v3
-  #    - name: Compute matrix
-  #      id: compute-matrix
-  #      run: |
-  #        cat ./ci/matrix.yaml
-  #        MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '.')
-  #        echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
-
-  #build:
-  #  needs: compute-matrix
-  #  if: github.repository == 'NVIDIA/cccl'
-  #  name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-  #  strategy:
-  #    fail-fast: false
-  #    max-parallel: 8
-  #    matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-  #  runs-on: linux-${{ matrix.cpu }}-cpu16
-  #  container:
-  #    options: -u root
-  #    image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-  #  permissions:
-  #    id-token: write 
-  #    contents: read 
-  #  steps:
-  #    - name: test
-  #      run: echo "hello"
-  #  steps:
-# #     - name: Verify compiler version
-# #       run: |
-# #         version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-# #         if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
-# #           echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
-# #           exit 1
-# #         fi
-# #         echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
-  #    - name: Get AWS credentials for sccache bucket
-  #      uses: aws-actions/configure-aws-credentials@v2
-  #      with:
-  #        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-  #        aws-region: us-east-2 
-  #        role-duration-seconds: 43200 # 12 hours
-  #    - uses: actions/checkout@v3
-  #      with:
-  #        fetch-depth: 0
-  #        persist-credentials: false
-  #    - name: Build Thrust tests
-  #      env:
-  #        SCCACHE_BUCKET: rapids-sccache-east
-  #        SCCACHE_REGION: us-east-2
-  #        SCCACHE_IDLE_TIMEOUT: 32768
-  #        SCCACHE_S3_USE_SSL: true
-  #        SCCACHE_S3_NO_CREDENTIALS: false
-  #      run: |
-  #        time bash ./ci/build_thrust.sh 
-  #        sccache -s
-  #    - name: tar Thrust build artifacts
-  #      run: |
-  #        tar czf thrust-build-artifacts.tar.gz ./thrust
-  #        echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
-  #    - name: Upload Thrust build artifacts
-  #      uses: actions/upload-artifact@v3
-  #      with:
-  #        name: thrust-build-artifacts
-  #        path: thrust-build-artifacts.tar.gz
+    #steps:
+#   #   - name: Verify compiler version
+#   #     run: |
+#   #       version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+#   #       if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
+#   #         echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+#   #         exit 1
+#   #       fi
+#   #       echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
+    #  - name: Get AWS credentials for sccache bucket
+    #    uses: aws-actions/configure-aws-credentials@v2
+    #    with:
+    #      role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+    #      aws-region: us-east-2 
+    #      role-duration-seconds: 43200 # 12 hours
+    #  - uses: actions/checkout@v3
+    #    with:
+    #      fetch-depth: 0
+    #      persist-credentials: false
+    #  - name: Build Thrust tests
+    #    env:
+    #      SCCACHE_BUCKET: rapids-sccache-east
+    #      SCCACHE_REGION: us-east-2
+    #      SCCACHE_IDLE_TIMEOUT: 32768
+    #      SCCACHE_S3_USE_SSL: true
+    #      SCCACHE_S3_NO_CREDENTIALS: false
+    #    run: |
+    #      time bash ./ci/build_thrust.sh 
+    #      sccache -s
+    #  - name: tar Thrust build artifacts
+    #    run: |
+    #      tar czf thrust-build-artifacts.tar.gz ./thrust
+    #      echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
+    #  - name: Upload Thrust build artifacts
+    #    uses: actions/upload-artifact@v3
+    #    with:
+    #      name: thrust-build-artifacts
+    #      path: thrust-build-artifacts.tar.gz
   #test:
   #  name: Run tests
   #  needs: build
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index b68b23c6649..35743a645fb 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -1,6 +1,10 @@
 #gcc
 
-std: [11, 14, 17, 20]
+std: 
+  - 11
+  - 14
+  - 17
+  - 20
 
 include:
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}

From d69c8e46cf280eddb32d4ca177e383c2f6a03e23 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 10 May 2023 10:49:37 -0500
Subject: [PATCH 114/395] Wrap matrix string with missing include

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 39b61e2fc36..9ef68d6c698 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ]')
+          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ] | {include: .}')
           echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   build:

From b0efcab2dfc7d171a8354d08573308a70d0fd40d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 10 May 2023 10:54:53 -0500
Subject: [PATCH 115/395] Add jobs for thrust/cub/libcu++

---
 .github/workflows/test.yml | 42 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9ef68d6c698..b11973e071b 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,10 +30,48 @@ jobs:
           MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ] | {include: .}')
           echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
-  build:
+  thrust:
     needs: compute-matrix
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Thrust (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: linux-${{ matrix.cpu }}-cpu4
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write 
+      contents: read 
+    steps:
+      - name: test
+        run: echo "hello"
+
+  libcudacxx:
+    needs: compute-matrix
+    if: github.repository == 'NVIDIA/cccl'
+    name: libcudacxx (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: linux-${{ matrix.cpu }}-cpu4
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write 
+      contents: read 
+    steps:
+      - name: test
+        run: echo "hello"
+
+  cub:
+    needs: compute-matrix
+    if: github.repository == 'NVIDIA/cccl'
+    name: CUB (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From 3791cc8e890cb998bbc3d96f4dee345f57816318 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 10 May 2023 14:42:34 -0500
Subject: [PATCH 116/395] Factor out common workflow logic to reusable
 workflow.

---
 .github/workflows/base.yml | 27 ++++++++++++++++++++++++
 .github/workflows/test.yml | 43 ++++----------------------------------
 2 files changed, 31 insertions(+), 39 deletions(-)
 create mode 100644 .github/workflows/base.yml

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
new file mode 100644
index 00000000000..08e89abfafb
--- /dev/null
+++ b/.github/workflows/base.yml
@@ -0,0 +1,27 @@
+name: Base CCCL workflow
+
+on:
+  workflow_call:
+    inputs:
+      matrix:
+        type: string
+        required: true
+
+jobs:
+  build:
+    if: github.repository == 'NVIDIA/cccl'
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJSON(inputs.matrix) }}
+    runs-on: linux-${{ matrix.cpu }}-cpu16
+    container:
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: test
+        run: echo "hello"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b11973e071b..ad9e5808e17 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,60 +32,25 @@ jobs:
 
   thrust:
     needs: compute-matrix
-    if: github.repository == 'NVIDIA/cccl'
-    name: Thrust (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: linux-${{ matrix.cpu }}-cpu4
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write 
-      contents: read 
+    uses: ./.github/workflows/base.yml
     steps:
       - name: test
         run: echo "hello"
 
   libcudacxx:
     needs: compute-matrix
-    if: github.repository == 'NVIDIA/cccl'
-    name: libcudacxx (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: linux-${{ matrix.cpu }}-cpu4
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write 
-      contents: read 
+    uses: ./.github/workflows/base.yml
     steps:
       - name: test
         run: echo "hello"
 
   cub:
     needs: compute-matrix
-    if: github.repository == 'NVIDIA/cccl'
-    name: CUB (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
-    runs-on: linux-${{ matrix.cpu }}-cpu4
-    container:
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write 
-      contents: read 
+    uses: ./.github/workflows/base.yml
     steps:
       - name: test
         run: echo "hello"
+
     #steps:
 #   #   - name: Verify compiler version
 #   #     run: |

From 5c84765c4e1fdb76a077577c40362437e626f5b2 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <avacanti@nvidia.com>
Date: Wed, 25 Jan 2023 15:05:40 -0500
Subject: [PATCH 117/395] Iniital monorepo setup.

---
 .gitignore                                 |   2 +
 .gitmodules                                |   9 ++
 CMakeLists.txt                             |  55 ++++++++
 cmake/CCCLAddSubdir.cmake                  |   6 +
 cmake/CCCLInstallRules.cmake               |  10 ++
 cub                                        |   1 +
 libcudacxx                                 |   1 +
 share/cmake/cccl/cccl-config-version.cmake |  13 ++
 share/cmake/cccl/cccl-config.cmake         |  59 +++++++++
 test/CMakeLists.txt                        |   1 +
 test/cmake/CMakeLists.txt                  |  81 ++++++++++++
 test/cmake/test_export/CMakeLists.txt      | 143 +++++++++++++++++++++
 test/cmake/test_export/version_check.cxx   |  20 +++
 thrust                                     |   1 +
 14 files changed, 402 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 .gitmodules
 create mode 100644 CMakeLists.txt
 create mode 100644 cmake/CCCLAddSubdir.cmake
 create mode 100644 cmake/CCCLInstallRules.cmake
 create mode 160000 cub
 create mode 160000 libcudacxx
 create mode 100644 share/cmake/cccl/cccl-config-version.cmake
 create mode 100644 share/cmake/cccl/cccl-config.cmake
 create mode 100644 test/CMakeLists.txt
 create mode 100644 test/cmake/CMakeLists.txt
 create mode 100644 test/cmake/test_export/CMakeLists.txt
 create mode 100644 test/cmake/test_export/version_check.cxx
 create mode 160000 thrust

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000000..aec8d8b240b
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.idea/
+*build*/
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 00000000000..1b1cc4ba9f5
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "libcudacxx"]
+	path = libcudacxx
+	url = ../libcudacxx.git
+[submodule "cub"]
+	path = cub
+	url = ../cub
+[submodule "thrust"]
+	path = thrust
+	url = ../thrust
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 00000000000..e40be2c4215
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,55 @@
+# 3.15 is the minimum for including the project with add_subdirectory.
+# 3.21 is the minimum for the developer build.
+cmake_minimum_required(VERSION 3.15)
+
+# Determine whether CCCL is the top-level project or included into
+# another project via add_subdirectory()
+if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
+  set(CCCL_TOPLEVEL_PROJECT ON)
+else()
+  set(CCCL_TOPLEVEL_PROJECT OFF)
+endif()
+
+# TODO find_package for add_subdir usecases, use components for subprojects.
+
+project(CCCL NONE)
+
+# Optionally include installation rules for non-top-level builds:
+option(CCCL_ENABLE_INSTALL_RULES "Enable installation of CCCL." ${CCCL_TOPLEVEL_PROJECT})
+if (CCCL_ENABLE_INSTALL_RULES)
+  include(cmake/CCCLInstallRules.cmake)
+endif()
+
+# Support adding Thrust to a parent project via add_subdirectory.
+# See examples/cmake/add_subdir/CMakeLists.txt for details.
+if (NOT CCCL_TOPLEVEL_PROJECT)
+  include(cmake/CCCLAddSubdir.cmake)
+  return()
+endif()
+
+# We require a higher cmake version for dev builds
+cmake_minimum_required(VERSION 3.21)
+
+option(CCCL_ENABLE_LIBCUDACXX "Enable the libcu++ developer build." ON)
+option(CCCL_ENABLE_CUB "Enable the CUB developer build." ON)
+option(CCCL_ENABLE_THRUST "Enable the Thrust developer build." ON)
+option(CCCL_ENABLE_TESTING "Enable CUDA C++ Core Library tests." ON)
+
+include(CTest)
+enable_testing()
+
+if (CCCL_ENABLE_LIBCUDACXX)
+  add_subdirectory(libcudacxx)
+endif()
+
+if (CCCL_ENABLE_CUB)
+  add_subdirectory(cub)
+endif()
+
+if (CCCL_ENABLE_THRUST)
+  add_subdirectory(thrust)
+endif()
+
+if (CCCL_ENABLE_TESTING)
+  add_subdirectory(test)
+endif()
diff --git a/cmake/CCCLAddSubdir.cmake b/cmake/CCCLAddSubdir.cmake
new file mode 100644
index 00000000000..b76238442d3
--- /dev/null
+++ b/cmake/CCCLAddSubdir.cmake
@@ -0,0 +1,6 @@
+find_package(CCCL REQUIRED CONFIG
+  NO_DEFAULT_PATH # Only check the explicit path in HINTS:
+  HINTS "${CCCL_SOURCE_DIR}"
+  COMPONENTS ${CCCL_REQUIRED_COMPONENTS}
+  OPTIONAL_COMPONENTS ${CCCL_OPTIONAL_COMPONENTS}
+)
diff --git a/cmake/CCCLInstallRules.cmake b/cmake/CCCLInstallRules.cmake
new file mode 100644
index 00000000000..39c22d37a58
--- /dev/null
+++ b/cmake/CCCLInstallRules.cmake
@@ -0,0 +1,10 @@
+# Bring in CMAKE_INSTALL_LIBDIR
+include(GNUInstallDirs)
+
+# CCCL has no installable binaries, no need to build before installing:
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)
+
+install(DIRECTORY "${CCCL_SOURCE_DIR}/share/cmake/cccl"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/"
+  PATTERN *.cmake.in EXCLUDE
+)
diff --git a/cub b/cub
new file mode 160000
index 00000000000..d46f513cdd5
--- /dev/null
+++ b/cub
@@ -0,0 +1 @@
+Subproject commit d46f513cdd5a8580e7a674d3a334b1fccc466233
diff --git a/libcudacxx b/libcudacxx
new file mode 160000
index 00000000000..ab5a255bae6
--- /dev/null
+++ b/libcudacxx
@@ -0,0 +1 @@
+Subproject commit ab5a255bae6b692716728862a7f97c740f1cfcd3
diff --git a/share/cmake/cccl/cccl-config-version.cmake b/share/cmake/cccl/cccl-config-version.cmake
new file mode 100644
index 00000000000..008257bc475
--- /dev/null
+++ b/share/cmake/cccl/cccl-config-version.cmake
@@ -0,0 +1,13 @@
+# TODO we should parse this from a header, and update to CalVer if that's still POR
+set(CCCL_VERSION_MAJOR 2)
+set(CCCL_VERSION_MINOR 1)
+set(CCCL_VERSION_PATCH 0)
+set(CCCL_VERSION_TWEAK 0)
+
+set(CCCL_VERSION "${CCCL_VERSION_MAJOR}.${CCCL_VERSION_MINOR}.${CCCL_VERSION_PATCH}.${CCCL_VERSION_TWEAK}")
+
+# TODO decide and implement versioning logic.
+set(PACKAGE_VERSION ${CCCL_VERSION})
+set(PACKAGE_VERSION_COMPATIBLE TRUE)
+set(PACKAGE_VERSION_EXACT TRUE)
+set(PACKAGE_VERSION_UNSUITABLE FALSE)
diff --git a/share/cmake/cccl/cccl-config.cmake b/share/cmake/cccl/cccl-config.cmake
new file mode 100644
index 00000000000..b2e7e211085
--- /dev/null
+++ b/share/cmake/cccl/cccl-config.cmake
@@ -0,0 +1,59 @@
+#
+# find_package(CCCL) config file.
+#
+# Imports the Thrust, CUB, and libcudacxx components of the NVIDIA
+# CUDA/C++ Core Libraries.
+
+unset(cccl_version) # TODO
+
+set(cccl_cmake_dir "${CMAKE_CURRENT_LIST_DIR}")
+
+if (${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+  set(cccl_quiet_flag "QUIET")
+else()
+  set(cccl_quiet_flag "")
+endif()
+
+foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
+  unset(req)
+  if (${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED_${component})
+    set(cccl_comp_required_flag "REQUIRED")
+  endif()
+
+  if(component STREQUAL "libcudacxx")
+    find_package(libcudacxx ${cccl_version} CONFIG
+      ${cccl_quiet_flag}
+      ${cccl_comp_required_flag}
+      NO_DEFAULT_PATH # Only check the explicit HINTS below:
+      HINTS
+        "${cccl_cmake_dir}/../../../libcudacxx/lib/cmake/" # Source layout (GitHub)
+        "${cccl_cmake_dir}/.."                             # Install layout
+      )
+  elseif(component STREQUAL "CUB")
+    find_package(CUB ${cccl_version} CONFIG
+      ${cccl_quiet_flag}
+      ${cccl_comp_required_flag}
+      NO_DEFAULT_PATH # Only check the explicit HINTS below:
+      HINTS
+        "${cccl_cmake_dir}/../../../cub/cub/cmake/" # Source layout (GitHub)
+        "${cccl_cmake_dir}/.."                      # Install layout
+    )
+  elseif(component STREQUAL "Thrust")
+    find_package(Thrust ${cccl_version} CONFIG
+      ${cccl_quiet_flag}
+      ${cccl_comp_required_flag}
+      NO_DEFAULT_PATH # Only check the explicit HINTS below:
+      HINTS
+        "${cccl_cmake_dir}/../../../thrust/thrust/cmake/" # Source layout (GitHub)
+        "${cccl_cmake_dir}/.."                            # Install layout
+    )
+  else()
+    message(FATAL_ERROR "Invalid CCCL component requested: '${component}'")
+  endif()
+endforeach()
+
+include(FindPackageHandleStandardArgs)
+if (NOT CCCL_CONFIG)
+  set(CCCL_CONFIG "${CMAKE_CURRENT_LIST_FILE}")
+endif()
+find_package_handle_standard_args(CCCL CONFIG_MODE)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
new file mode 100644
index 00000000000..24c8f171635
--- /dev/null
+++ b/test/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(cmake)
diff --git a/test/cmake/CMakeLists.txt b/test/cmake/CMakeLists.txt
new file mode 100644
index 00000000000..c8f3aa32b92
--- /dev/null
+++ b/test/cmake/CMakeLists.txt
@@ -0,0 +1,81 @@
+set(cmake_opts
+  -D "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+  -D "CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}"
+  -D "CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+)
+
+# Temporary installation prefix for tests against installed project:
+set(tmp_install_prefix "${CMAKE_CURRENT_BINARY_DIR}/test_install")
+
+# Add a build-and-test CTest.
+# - full_test_name_var will be set to the full name of the test.
+# - subdir is the relative path to the test project directory.
+# - test_id is used to generate a unique name for this test, allowing the
+#   subdir to be reused.
+# - Any additional args will be passed to the project configure step.
+function(cccl_add_compile_test full_test_name_var subdir test_id)
+  set(test_name cccl.test.cmake.${subdir}.${test_id})
+  set(src_dir "${CMAKE_CURRENT_SOURCE_DIR}/${subdir}")
+  set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/${subdir}/${test_id}")
+  add_test(NAME ${test_name}
+    COMMAND "${CMAKE_CTEST_COMMAND}"
+      --build-and-test "${src_dir}" "${build_dir}"
+      --build-generator "${CMAKE_GENERATOR}"
+      --build-options
+        ${cmake_opts}
+        ${ARGN}
+      --test-command "${CMAKE_CTEST_COMMAND}" --output-on-failure
+  )
+  set(${full_test_name_var} ${test_name} PARENT_SCOPE)
+endfunction()
+
+foreach (root_type IN ITEMS SOURCE INSTALL)
+  if (root_type STREQUAL "INSTALL")
+    set(cccl_root "${tmp_install_prefix}")
+  else()
+    set(cccl_root "${CCCL_SOURCE_DIR}")
+  endif()
+
+  foreach (components IN ITEMS DEFAULT Thrust CUB libcudacxx)
+    set(package_types CCCL)
+    if (NOT components STREQUAL "DEFAULT")
+      list(APPEND package_types NATIVE)
+    endif()
+    if (root_type STREQUAL "SOURCE")
+      list(APPEND package_types SUBDIR)
+    endif()
+    foreach (package_type IN LISTS package_types)
+      string(TOLOWER "${root_type}.${package_type}.${components}" suffix)
+      cccl_add_compile_test(test_name
+        test_export
+        "${suffix}"
+        -D "CCCL_ROOT=${cccl_root}"
+        -D "ROOT_TYPE=${root_type}"
+        -D "COMPONENTS=${components}"
+        -D "PACKAGE_TYPE=${package_type}"
+      )
+
+      if (root_type STREQUAL "INSTALL")
+        set_tests_properties(${test_name} PROPERTIES FIXTURES_REQUIRED install_tree)
+      endif()
+    endforeach() # package_type
+  endforeach() # components
+endforeach() # root_type
+
+################################################################################
+# Install tree fixtures
+add_test(NAME cccl.test.cmake.install_tree.install
+  COMMAND "${CMAKE_COMMAND}"
+    --install "${CCCL_BINARY_DIR}"
+    --prefix "${tmp_install_prefix}"
+)
+set_tests_properties(cccl.test.cmake.install_tree.install PROPERTIES
+  FIXTURES_SETUP install_tree
+)
+
+add_test(NAME cccl.test.cmake.install_tree.cleanup
+  COMMAND "${CMAKE_COMMAND}" -E rm -rf "${tmp_install_prefix}"
+)
+set_tests_properties(cccl.test.cmake.install_tree.cleanup PROPERTIES
+  FIXTURES_CLEANUP install_tree
+)
diff --git a/test/cmake/test_export/CMakeLists.txt b/test/cmake/test_export/CMakeLists.txt
new file mode 100644
index 00000000000..56e3ffb7f11
--- /dev/null
+++ b/test/cmake/test_export/CMakeLists.txt
@@ -0,0 +1,143 @@
+# Test the CMake packages for CCCL and all subprojects.
+#
+# Parameters:
+# - CCCL_ROOT [Path] Root of the CCCL repo, or an installation root.
+# - ROOT_TYPE [String] {SOURCE | INSTALL} Whether CCCL_ROOT is an
+#   installation prefix or the source root.
+# - COMPONENTS [StringList] {Thrust CUB libcudacxx} Which CCCL subprojects
+#   should be found.
+# - PACKAGE_TYPE [String] {CCCL | NATIVE | SUBDIR}:
+#   - CCCL -> `find_package(CCCL COMPONENTS <subproject>)`
+#   - NATIVE -> `find_package(<subproject>)`
+#   - SUBDIR -> `set(CCCL_REQUIRED_COMPONENTS <subproject>)`
+#               `add_subdirectory(${cccl_root})`
+
+
+cmake_minimum_required(VERSION 3.21)
+project(CCCLTestExport CXX)
+
+include(CTest)
+enable_testing()
+
+set(CCCL_ROOT "" CACHE PATH
+  "Root of the CCCL repo, or an installation root.")
+set(ROOT_TYPE "" CACHE STRING
+  "{SOURCE | INSTALL} Whether CCCL_ROOT is an install prefix or source root.")
+set_property(CACHE ROOT_TYPE PROPERTY STRINGS SOURCE INSTALL)
+set(COMPONENTS "" CACHE STRING
+  "DEFAULT for no components, or semi-colon delimited list of Thrust, CUB, and/or libcudacxx.")
+set(PACKAGE_TYPE "" CACHE STRING
+  "CCCL: Find CCCL with subpackages as components; NATIVE: Find subpackages directly; SUBDIR: add_subdirectory(${CCCL_ROOT}")
+set_property(CACHE PACKAGE_TYPE PROPERTY STRINGS CCCL NATIVE SUBDIR)
+
+message(STATUS "CCCL_ROOT=${CCCL_ROOT}")
+message(STATUS "ROOT_TYPE=${ROOT_TYPE}")
+message(STATUS "COMPONENTS=${COMPONENTS}")
+message(STATUS "PACKAGE_TYPE=${PACKAGE_TYPE}")
+
+# TODO `ROOT_TYPE` probably won't be needed after the configs are all moved
+# to the CCCL repo.
+if (ROOT_TYPE STREQUAL "SOURCE")
+  cmake_path(APPEND CCCL_ROOT thrust OUTPUT_VARIABLE Thrust_ROOT)
+  cmake_path(APPEND CCCL_ROOT cub OUTPUT_VARIABLE CUB_ROOT)
+  cmake_path(APPEND CCCL_ROOT libcudacxx OUTPUT_VARIABLE libcudacxx_ROOT)
+elseif (ROOT_TYPE STREQUAL "INSTALL")
+  set(Thrust_ROOT "${CCCL_ROOT}")
+  set(CUB_ROOT "${CCCL_ROOT}")
+  set(libcudacxx_ROOT "${CCCL_ROOT}")
+else()
+  message(FATAL_ERROR "Invalid ROOT_TYPE: ${ROOT_TYPE}")
+endif()
+
+message(STATUS "Thrust_ROOT=${Thrust_ROOT}")
+message(STATUS "CUB_ROOT=${CUB_ROOT}")
+message(STATUS "libcudacxx_ROOT=${libcudacxx_ROOT}")
+
+function(do_find_package pkg_name pkg_prefix)
+  list(APPEND arg_list
+    REQUIRED
+    ${ARGN}
+    NO_DEFAULT_PATH
+    HINTS "${pkg_prefix}"
+  )
+  list(JOIN arg_list " " arg_str)
+  message(STATUS "Executing: find_package(${pkg_name} ${arg_str})")
+  find_package(${pkg_name} ${arg_list})
+  if (NOT ${pkg_name}_FOUND)
+    message(FATAL_ERROR "Failed: find_package(${pkg_name} ${arg_str})")
+  endif()
+endfunction()
+
+# Run find package with the requested configuration:
+if (PACKAGE_TYPE STREQUAL "CCCL")
+  if (COMPONENTS STREQUAL "DEFAULT")
+    do_find_package(CCCL "${CCCL_ROOT}")
+  else()
+    do_find_package(CCCL "${CCCL_ROOT}" COMPONENTS ${COMPONENTS})
+  endif()
+elseif(PACKAGE_TYPE STREQUAL "NATIVE")
+  if (COMPONENTS STREQUAL "DEFAULT")
+    message(FATAL_ERROR "COMPONENTS=DEFAULT incompatible with PACKAGE_TYPE=NATIVE")
+  endif()
+  foreach (component IN LISTS COMPONENTS)
+    do_find_package(${component} "${${component}_ROOT}")
+  endforeach()
+elseif(PACKAGE_TYPE STREQUAL "SUBDIR")
+  if (COMPONENTS STREQUAL "DEFAULT")
+    set(CCCL_REQUIRED_COMPONENTS)
+  else()
+    set(CCCL_REQUIRED_COMPONENTS ${COMPONENTS})
+  endif()
+  add_subdirectory("${CCCL_ROOT}" "${CMAKE_CURRENT_BINARY_DIR}/subdir")
+else()
+  message(FATAL_ERROR "Invalid PACKAGE_TYPE: ${PACKAGE_TYPE}")
+endif()
+
+if (COMPONENTS STREQUAL "DEFAULT")
+  set(COMPONENTS libcudacxx CUB Thrust)
+endif()
+
+foreach (component IN LISTS COMPONENTS)
+  set(test_target version_check.${component})
+  set(component_target "${component}::${component}")
+  add_executable(${test_target} version_check.cxx)
+  target_link_libraries(${test_target} PRIVATE ${component_target})
+  add_test(NAME ${test_target} COMMAND ${test_target})
+
+  if (component STREQUAL "libcudacxx")
+    find_package(libcudacxx) # Make sure version vars are in scope
+    # TODO Should this be a genex? Will these stay correct as versions change?
+    math(EXPR component_cmake_version
+      "(${libcudacxx_VERSION_MAJOR} * 1000000) +
+        ${libcudacxx_VERSION_MINOR} * 1000 +
+        ${libcudacxx_VERSION_PATCH}")
+    target_compile_definitions(${test_target} PRIVATE
+      "VERSION_HEADER=cuda/std/version"
+      "VERSION_MACRO=_LIBCUDACXX_CUDA_API_VERSION"
+      "EXPECTED_VERSION=${component_cmake_version}")
+  elseif (component STREQUAL "CUB")
+    find_package(CUB) # Make sure version vars are in scope
+    # TODO Should this be a genex? Will these stay correct as versions change?
+    math(EXPR component_cmake_version
+      "(${CUB_VERSION_MAJOR} * 100000) +
+        ${CUB_VERSION_MINOR} * 100 +
+        ${CUB_VERSION_PATCH}")
+    target_compile_definitions(${test_target} PRIVATE
+      "VERSION_HEADER=cub/version.cuh"
+      "VERSION_MACRO=CUB_VERSION"
+      "EXPECTED_VERSION=${component_cmake_version}")
+  elseif (component STREQUAL "Thrust")
+    find_package(Thrust) # Make sure version vars are in scope
+    # TODO Should this be a genex? Will these stay correct as versions change?
+    math(EXPR component_cmake_version
+      "(${THRUST_VERSION_MAJOR} * 100000) +
+        ${THRUST_VERSION_MINOR} * 100 +
+        ${THRUST_VERSION_PATCH}")
+    target_compile_definitions(${test_target} PRIVATE
+      "VERSION_HEADER=thrust/version.h"
+      "VERSION_MACRO=THRUST_VERSION"
+      "EXPECTED_VERSION=${component_cmake_version}")
+  else()
+    message(FATAL_ERROR "Valid COMPONENTS are (case-sensitive): Thrust;CUB;libcudacxx")
+  endif()
+endforeach()
diff --git a/test/cmake/test_export/version_check.cxx b/test/cmake/test_export/version_check.cxx
new file mode 100644
index 00000000000..c09e8d9cd1f
--- /dev/null
+++ b/test/cmake/test_export/version_check.cxx
@@ -0,0 +1,20 @@
+// Compile with:
+//   -DVERSION_HEADER=include/path/for/version.h
+//   -DEXPECTED_VERSION=XXYYZZ
+//   -DVERSION_MACRO=PROJECT_VERSION
+
+#define HEADER <VERSION_HEADER>
+#include HEADER
+
+#include <cstdio>
+
+#define DETECTED_VERSION VERSION_MACRO
+
+int main()
+{
+  printf("Expected version: %d\n"
+         "Detected version: %d\n",
+         EXPECTED_VERSION,
+         VERSION_MACRO);
+  return EXPECTED_VERSION == DETECTED_VERSION ? 0 : 1;
+}
diff --git a/thrust b/thrust
new file mode 160000
index 00000000000..c1d4edd7fab
--- /dev/null
+++ b/thrust
@@ -0,0 +1 @@
+Subproject commit c1d4edd7fabc48462525b3cfb40a1629d6af007a

From d88667b1d87a2e46b74524c7e8b7df5c2fe5a450 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Tue, 16 May 2023 11:32:37 -0400
Subject: [PATCH 118/395] Add semantic version checks for CCCL package.

---
 libcudacxx                                 |  2 +-
 share/cmake/cccl/cccl-config-version.cmake | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/libcudacxx b/libcudacxx
index ab5a255bae6..c9a2d691c52 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit ab5a255bae6b692716728862a7f97c740f1cfcd3
+Subproject commit c9a2d691c520bdbaa5783db74b08c2c51435d80d
diff --git a/share/cmake/cccl/cccl-config-version.cmake b/share/cmake/cccl/cccl-config-version.cmake
index 008257bc475..97a42b140a1 100644
--- a/share/cmake/cccl/cccl-config-version.cmake
+++ b/share/cmake/cccl/cccl-config-version.cmake
@@ -1,13 +1,23 @@
-# TODO we should parse this from a header, and update to CalVer if that's still POR
+# TODO How should we encode this as a source of truth?
 set(CCCL_VERSION_MAJOR 2)
-set(CCCL_VERSION_MINOR 1)
+set(CCCL_VERSION_MINOR 2)
 set(CCCL_VERSION_PATCH 0)
 set(CCCL_VERSION_TWEAK 0)
 
 set(CCCL_VERSION "${CCCL_VERSION_MAJOR}.${CCCL_VERSION_MINOR}.${CCCL_VERSION_PATCH}.${CCCL_VERSION_TWEAK}")
 
-# TODO decide and implement versioning logic.
 set(PACKAGE_VERSION ${CCCL_VERSION})
-set(PACKAGE_VERSION_COMPATIBLE TRUE)
-set(PACKAGE_VERSION_EXACT TRUE)
+set(PACKAGE_VERSION_COMPATIBLE FALSE)
+set(PACKAGE_VERSION_EXACT FALSE)
 set(PACKAGE_VERSION_UNSUITABLE FALSE)
+
+# Semantic versioning:
+if(PACKAGE_VERSION VERSION_GREATER_EQUAL PACKAGE_FIND_VERSION)
+  if(CCCL_VERSION_MAJOR VERSION_EQUAL PACKAGE_FIND_VERSION_MAJOR)
+    set(PACKAGE_VERSION_COMPATIBLE TRUE)
+  endif()
+
+  if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION)
+    set(PACKAGE_VERSION_EXACT TRUE)
+  endif()
+endif()

From 7e4cbeb11eb99e3baa8366bc87aee4acea27d86d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 08:49:27 -0500
Subject: [PATCH 119/395] Remove steps when using reusable workflow.

---
 .github/workflows/test.yml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ad9e5808e17..6c3c1ed66c1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,23 +33,14 @@ jobs:
   thrust:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
-    steps:
-      - name: test
-        run: echo "hello"
 
   libcudacxx:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
-    steps:
-      - name: test
-        run: echo "hello"
 
   cub:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
-    steps:
-      - name: test
-        run: echo "hello"
 
     #steps:
 #   #   - name: Verify compiler version

From f7091f139e4c582af311238bd1dfae29c38babc3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 08:58:01 -0500
Subject: [PATCH 120/395] Pass matrix to thrust/cub/libcu++ jobs.

---
 .github/workflows/test.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6c3c1ed66c1..8b668534d32 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,14 +33,21 @@ jobs:
   thrust:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
+    with:
+      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+
 
   libcudacxx:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
+    with:
+      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
 
   cub:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
+    with:
+      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
 
     #steps:
 #   #   - name: Verify compiler version

From e0b0b1fc410775688a272371a0274328c145a1b4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:01:04 -0500
Subject: [PATCH 121/395] Remove Ubuntu version from job name.

---
 .github/workflows/base.yml | 2 +-
 .github/workflows/test.yml | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 08e89abfafb..4af09c09e3d 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -10,7 +10,7 @@ on:
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, Ubuntu ${{ matrix.ubuntu }}, CPU ${{matrix.cpu}})
+    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, CPU ${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8b668534d32..282ff1c6b30 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,7 +36,6 @@ jobs:
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
 
-
   libcudacxx:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml

From 787e44b0d2834e7fd681bd58447e99ca401b8f14 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:10:44 -0500
Subject: [PATCH 122/395] Add AWS credentials and build_script input to base
 workflow.

---
 .github/workflows/base.yml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 4af09c09e3d..4913da04993 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -6,6 +6,9 @@ on:
       matrix:
         type: string
         required: true
+      build_script:
+        type: string
+        required: true
 
 jobs:
   build:
@@ -23,5 +26,25 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: test
-        run: echo "hello"
+    - name: Get AWS credentials for sccache bucket
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+        aws-region: us-east-2 
+        role-duration-seconds: 43200 # 12 hours
+    - name: Checkout repo
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        persist-credentials: false
+    - name: Run build script
+        env:
+          SCCACHE_BUCKET: rapids-sccache-east
+          SCCACHE_REGION: us-east-2
+          SCCACHE_IDLE_TIMEOUT: 32768
+          SCCACHE_S3_USE_SSL: true
+          SCCACHE_S3_NO_CREDENTIALS: false
+      run: |
+        chmod +x ${{ inputs.build_script }}
+        ./${{ inputs.build_script }}
+        sccache -s

From 024d65b1e0c3ee8e2068a2310e67e408a2fa7149 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:11:03 -0500
Subject: [PATCH 123/395] Pass build script to each library job.

---
 .github/workflows/test.yml |  3 +++
 ci/build_cub.sh            | 26 ++++++++++++++++++++++++++
 ci/build_libcudacxx.sh     | 26 ++++++++++++++++++++++++++
 ci/build_thrust.sh         |  8 +++++---
 4 files changed, 60 insertions(+), 3 deletions(-)
 create mode 100755 ci/build_cub.sh
 create mode 100755 ci/build_libcudacxx.sh

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 282ff1c6b30..445f6e601af 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -35,18 +35,21 @@ jobs:
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_script: ./ci/build_thrust.sh
 
   libcudacxx:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_script: ./ci/build_libcudacxx.sh
 
   cub:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_script: ./ci/build_cub.sh
 
     #steps:
 #   #   - name: Verify compiler version
diff --git a/ci/build_cub.sh b/ci/build_cub.sh
new file mode 100755
index 00000000000..2782951996b
--- /dev/null
+++ b/ci/build_cub.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "build cub"
+
+# Require a compiler to be specified
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
+
+# First argument is the CXX compiler
+#CXX_COMPILER=$1
+
+# Clone Thrust repository 
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+
+# Configure Thrust
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+
+
+# Build Thrust tests
+#cmake --build thrust/build
+
+
+# Run Thrust tests
+#ctest --output-on-failure
diff --git a/ci/build_libcudacxx.sh b/ci/build_libcudacxx.sh
new file mode 100755
index 00000000000..106049915f4
--- /dev/null
+++ b/ci/build_libcudacxx.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "build libcudacxx"
+
+# Require a compiler to be specified
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
+
+# First argument is the CXX compiler
+#CXX_COMPILER=$1
+
+# Clone Thrust repository 
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+
+# Configure Thrust
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+
+
+# Build Thrust tests
+#cmake --build thrust/build
+
+
+# Run Thrust tests
+#ctest --output-on-failure
diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 042715ab178..4f901e064b4 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+echo "build thrust"
+
 # Require a compiler to be specified
 #if [ $# -ne 1 ]; then
 #    echo "Usage: $0 <CXX_COMPILER>"
@@ -10,14 +12,14 @@
 #CXX_COMPILER=$1
 
 # Clone Thrust repository 
-git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
 
 # Configure Thrust
-cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
 
 
 # Build Thrust tests
-cmake --build thrust/build
+#cmake --build thrust/build
 
 
 # Run Thrust tests

From 4d01c7283ee73de9ed6aff5b6f43a39d328330c3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:13:10 -0500
Subject: [PATCH 124/395] Quote build script paths.

---
 .github/workflows/test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 445f6e601af..bb3e4f4fb94 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -35,21 +35,21 @@ jobs:
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
-      build_script: ./ci/build_thrust.sh
+      build_script: "./ci/build_thrust.sh"
 
   libcudacxx:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
-      build_script: ./ci/build_libcudacxx.sh
+      build_script: "./ci/build_libcudacxx.sh"
 
   cub:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
-      build_script: ./ci/build_cub.sh
+      build_script: "./ci/build_cub.sh"
 
     #steps:
 #   #   - name: Verify compiler version

From bac89ac1e65a69c80c9948f1a55901cbd5f6cce3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:14:38 -0500
Subject: [PATCH 125/395] Fix indentation.

---
 .github/workflows/base.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 4913da04993..c77304d4bff 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -38,7 +38,7 @@ jobs:
         fetch-depth: 0
         persist-credentials: false
     - name: Run build script
-        env:
+      env:
           SCCACHE_BUCKET: rapids-sccache-east
           SCCACHE_REGION: us-east-2
           SCCACHE_IDLE_TIMEOUT: 32768

From 5d42b8de0f517b3a4bf16e9e9b23ab390a7df97b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:23:50 -0500
Subject: [PATCH 126/395] Remove accidental thrust submodule.

---
 ci/thrust | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 ci/thrust

diff --git a/ci/thrust b/ci/thrust
deleted file mode 160000
index 02931a309be..00000000000
--- a/ci/thrust
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 02931a309bee769853088b79b4e3ab1c0bd2336c

From 793d9349612fb67465c611dc3e9c81a69d981f3c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:41:44 -0500
Subject: [PATCH 127/395] Add test job to base workflow.

---
 .github/workflows/base.yml | 52 ++++++++++++++++++++++++++++++++++----
 .github/workflows/test.yml |  3 +++
 ci/test_cub.sh             | 26 +++++++++++++++++++
 ci/test_libcudacxx.sh      | 26 +++++++++++++++++++
 ci/test_thrust.sh          | 26 +++++++++++++++++++
 5 files changed, 128 insertions(+), 5 deletions(-)
 create mode 100755 ci/test_cub.sh
 create mode 100755 ci/test_libcudacxx.sh
 create mode 100755 ci/test_thrust.sh

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index c77304d4bff..7952fc6b99b 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -3,10 +3,13 @@ name: Base CCCL workflow
 on:
   workflow_call:
     inputs:
-      matrix:
+      matrix: # The matrix to use for this job
         type: string
         required: true
-      build_script:
+      build_script: # The script to run for the build step
+        type: string
+        required: true
+      test_script: 
         type: string
         required: true
 
@@ -19,10 +22,10 @@ jobs:
       max-parallel: 8
       matrix: ${{ fromJSON(inputs.matrix) }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
-    container:
+    container: # Fetch the container image to use for the job
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
+    permissions: # Set permissions needed for sccache/AWS credentials
       id-token: write
       contents: read
     steps:
@@ -38,7 +41,7 @@ jobs:
         fetch-depth: 0
         persist-credentials: false
     - name: Run build script
-      env:
+      env: # Set environment variables for sccache
           SCCACHE_BUCKET: rapids-sccache-east
           SCCACHE_REGION: us-east-2
           SCCACHE_IDLE_TIMEOUT: 32768
@@ -48,3 +51,42 @@ jobs:
         chmod +x ${{ inputs.build_script }}
         ./${{ inputs.build_script }}
         sccache -s
+  test:
+    if: github.repository == 'NVIDIA/cccl'
+    name: Test (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, CPU ${{matrix.cpu}})
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJSON(inputs.matrix) }}
+    runs-on: linux-${{matrix.cpu}}-gpu-v100-latest-1
+    container: # Fetch the container image to use for the job
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    permissions: # Set permissions needed for sccache/AWS credentials
+      id-token: write
+      contents: read
+    steps:
+    - name: Get AWS credentials for sccache bucket
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+        aws-region: us-east-2 
+        role-duration-seconds: 43200 # 12 hours
+    - name: Checkout repo
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        persist-credentials: false
+    - name: Run test script
+      env: # Set environment variables for sccache
+          SCCACHE_BUCKET: rapids-sccache-east
+          SCCACHE_REGION: us-east-2
+          SCCACHE_IDLE_TIMEOUT: 32768
+          SCCACHE_S3_USE_SSL: true
+          SCCACHE_S3_NO_CREDENTIALS: false
+      run: |
+        chmod +x ${{ inputs.test_script }}
+        ./${{ inputs.test_script }}
+        sccache -s
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bb3e4f4fb94..8f05f45471f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,6 +36,7 @@ jobs:
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
       build_script: "./ci/build_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
 
   libcudacxx:
     needs: compute-matrix
@@ -43,6 +44,7 @@ jobs:
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
       build_script: "./ci/build_libcudacxx.sh"
+      test_script: "./ci/test_libcudacxx.sh"
 
   cub:
     needs: compute-matrix
@@ -50,6 +52,7 @@ jobs:
     with:
       matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
       build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
 
     #steps:
 #   #   - name: Verify compiler version
diff --git a/ci/test_cub.sh b/ci/test_cub.sh
new file mode 100755
index 00000000000..2782951996b
--- /dev/null
+++ b/ci/test_cub.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "build cub"
+
+# Require a compiler to be specified
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
+
+# First argument is the CXX compiler
+#CXX_COMPILER=$1
+
+# Clone Thrust repository 
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+
+# Configure Thrust
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+
+
+# Build Thrust tests
+#cmake --build thrust/build
+
+
+# Run Thrust tests
+#ctest --output-on-failure
diff --git a/ci/test_libcudacxx.sh b/ci/test_libcudacxx.sh
new file mode 100755
index 00000000000..106049915f4
--- /dev/null
+++ b/ci/test_libcudacxx.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "build libcudacxx"
+
+# Require a compiler to be specified
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
+
+# First argument is the CXX compiler
+#CXX_COMPILER=$1
+
+# Clone Thrust repository 
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+
+# Configure Thrust
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+
+
+# Build Thrust tests
+#cmake --build thrust/build
+
+
+# Run Thrust tests
+#ctest --output-on-failure
diff --git a/ci/test_thrust.sh b/ci/test_thrust.sh
new file mode 100755
index 00000000000..4f901e064b4
--- /dev/null
+++ b/ci/test_thrust.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "build thrust"
+
+# Require a compiler to be specified
+#if [ $# -ne 1 ]; then
+#    echo "Usage: $0 <CXX_COMPILER>"
+#    exit 1
+#fi
+
+# First argument is the CXX compiler
+#CXX_COMPILER=$1
+
+# Clone Thrust repository 
+#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+
+# Configure Thrust
+#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+
+
+# Build Thrust tests
+#cmake --build thrust/build
+
+
+# Run Thrust tests
+#ctest --output-on-failure

From 0fc0795c252722e368a16016c6299e0fa9f0d5bf Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:43:00 -0500
Subject: [PATCH 128/395] Make test job depend on build.

---
 .github/workflows/base.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 7952fc6b99b..4998ab731ec 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -52,6 +52,7 @@ jobs:
         ./${{ inputs.build_script }}
         sccache -s
   test:
+    needs: build
     if: github.repository == 'NVIDIA/cccl'
     name: Test (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, CPU ${{matrix.cpu}})
     strategy:

From 592fde43598ad4a6ed6f323cf060179a1a387e11 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:45:47 -0500
Subject: [PATCH 129/395] Shorten name of the build/test jobs.

---
 .github/workflows/base.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 4998ab731ec..03f80c13f4e 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, CPU ${{matrix.cpu}})
+    name: Build/(CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8
@@ -54,7 +54,7 @@ jobs:
   test:
     needs: build
     if: github.repository == 'NVIDIA/cccl'
-    name: Test (CUDA ${{matrix.cuda}}, ${{ matrix.compiler.name }}-${{ matrix.compiler.version }}, C++-${{matrix.std}}, CPU ${{matrix.cpu}})
+    name: Test/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From 7810643dff48d3b108ebd9fa351e229cc23e18cf Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 09:47:33 -0500
Subject: [PATCH 130/395] Remove parens from build name.

---
 .github/workflows/base.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 03f80c13f4e..d1b463b1131 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -16,7 +16,7 @@ on:
 jobs:
   build:
     if: github.repository == 'NVIDIA/cccl'
-    name: Build/(CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}})
+    name: Build/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     strategy:
       fail-fast: false
       max-parallel: 8

From 1f8fb932ab4ef5587375b270178afaa64b6fe374 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Fri, 26 May 2023 11:17:35 -0400
Subject: [PATCH 131/395] Update CMake option docs in subprojects.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index d46f513cdd5..6477d389833 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit d46f513cdd5a8580e7a674d3a334b1fccc466233
+Subproject commit 6477d389833321bc0ada92fedc3a5f8d76d6d0aa
diff --git a/thrust b/thrust
index c1d4edd7fab..7740478ceaf 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit c1d4edd7fabc48462525b3cfb40a1629d6af007a
+Subproject commit 7740478ceaf375c34b8e1bc280d94493e710ab96

From 93bb4bc2926fb19055de56bfdaa05894e8c37d6c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 10:20:33 -0500
Subject: [PATCH 132/395] Extract sccache config into reusable workflow.

---
 .github/workflows/base.yml                   | 28 +++-----------------
 .github/workflows/configure_cccl_sccache.yml | 18 +++++++++++++
 2 files changed, 22 insertions(+), 24 deletions(-)
 create mode 100644 .github/workflows/configure_cccl_sccache.yml

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index d1b463b1131..3b37774ae87 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -29,24 +29,14 @@ jobs:
       id-token: write
       contents: read
     steps:
-    - name: Get AWS credentials for sccache bucket
-      uses: aws-actions/configure-aws-credentials@v2
-      with:
-        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-        aws-region: us-east-2 
-        role-duration-seconds: 43200 # 12 hours
+    - name: Configure credentials and environment variables for sccache
+      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Checkout repo
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         persist-credentials: false
     - name: Run build script
-      env: # Set environment variables for sccache
-          SCCACHE_BUCKET: rapids-sccache-east
-          SCCACHE_REGION: us-east-2
-          SCCACHE_IDLE_TIMEOUT: 32768
-          SCCACHE_S3_USE_SSL: true
-          SCCACHE_S3_NO_CREDENTIALS: false
       run: |
         chmod +x ${{ inputs.build_script }}
         ./${{ inputs.build_script }}
@@ -69,24 +59,14 @@ jobs:
       id-token: write
       contents: read
     steps:
-    - name: Get AWS credentials for sccache bucket
-      uses: aws-actions/configure-aws-credentials@v2
-      with:
-        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-        aws-region: us-east-2 
-        role-duration-seconds: 43200 # 12 hours
+    - name: Configure credentials and environment variables for sccache
+      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Checkout repo
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         persist-credentials: false
     - name: Run test script
-      env: # Set environment variables for sccache
-          SCCACHE_BUCKET: rapids-sccache-east
-          SCCACHE_REGION: us-east-2
-          SCCACHE_IDLE_TIMEOUT: 32768
-          SCCACHE_S3_USE_SSL: true
-          SCCACHE_S3_NO_CREDENTIALS: false
       run: |
         chmod +x ${{ inputs.test_script }}
         ./${{ inputs.test_script }}
diff --git a/.github/workflows/configure_cccl_sccache.yml b/.github/workflows/configure_cccl_sccache.yml
new file mode 100644
index 00000000000..cf66e4621e9
--- /dev/null
+++ b/.github/workflows/configure_cccl_sccache.yml
@@ -0,0 +1,18 @@
+name: Set up AWS credentials and environment variables for sccache
+
+on:
+  workflow_call:
+    steps:
+      - name: Get AWS credentials for sccache bucket
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+          aws-region: us-east-2 
+          role-duration-seconds: 43200 # 12 hours)
+      - name: Set environment variables
+        run: |
+          echo "SCCACHE_BUCKET=rapids-sccache-east" >> $GITHUB_ENV
+          echo "SCCACHE_REGION=us-east-2" >> $GITHUB_ENV
+          echo "SCCACHE_IDLE_TIMEOUT=32768" >> $GITHUB_ENV
+          echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV
+          echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV
\ No newline at end of file

From 72f632263525b05487cd9ca1d7277c0395111c4c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 10:31:18 -0500
Subject: [PATCH 133/395] The sccache config step needs to be after checkout.

---
 .github/workflows/base.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 3b37774ae87..473d29ed368 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -29,13 +29,13 @@ jobs:
       id-token: write
       contents: read
     steps:
-    - name: Configure credentials and environment variables for sccache
-      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Checkout repo
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         persist-credentials: false
+    - name: Configure credentials and environment variables for sccache
+      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Run build script
       run: |
         chmod +x ${{ inputs.build_script }}
@@ -59,13 +59,13 @@ jobs:
       id-token: write
       contents: read
     steps:
-    - name: Configure credentials and environment variables for sccache
-      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Checkout repo
       uses: actions/checkout@v3
       with:
         fetch-depth: 0
         persist-credentials: false
+    - name: Configure credentials and environment variables for sccache
+      uses: ./.github/workflows/configure_cccl_sccache.yml
     - name: Run test script
       run: |
         chmod +x ${{ inputs.test_script }}

From d4ce78ca5438bf4da89ff63cc18ea4219ce0a64f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 10:43:47 -0500
Subject: [PATCH 134/395] And EOL

---
 .github/workflows/configure_cccl_sccache.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/configure_cccl_sccache.yml b/.github/workflows/configure_cccl_sccache.yml
index cf66e4621e9..2e26f1a498a 100644
--- a/.github/workflows/configure_cccl_sccache.yml
+++ b/.github/workflows/configure_cccl_sccache.yml
@@ -15,4 +15,4 @@ on:
           echo "SCCACHE_REGION=us-east-2" >> $GITHUB_ENV
           echo "SCCACHE_IDLE_TIMEOUT=32768" >> $GITHUB_ENV
           echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV
-          echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV
\ No newline at end of file
+          echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV

From aa368976e3ecf66fee8f53d60789da0f384beece Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 11:10:22 -0500
Subject: [PATCH 135/395] Apparently it needs to be a "composite action" not a
 reusable workflow.

---
 .../actions/configure_cccl_sccache/action.yml  | 18 ++++++++++++++++++
 .github/workflows/base.yml                     |  4 ++--
 .github/workflows/configure_cccl_sccache.yml   | 18 ------------------
 3 files changed, 20 insertions(+), 20 deletions(-)
 create mode 100644 .github/actions/configure_cccl_sccache/action.yml
 delete mode 100644 .github/workflows/configure_cccl_sccache.yml

diff --git a/.github/actions/configure_cccl_sccache/action.yml b/.github/actions/configure_cccl_sccache/action.yml
new file mode 100644
index 00000000000..b47de57ba9d
--- /dev/null
+++ b/.github/actions/configure_cccl_sccache/action.yml
@@ -0,0 +1,18 @@
+name: Set up AWS credentials and environment variables for sccache
+description: "Set up AWS credentials and environment variables for sccache"
+runs:
+  using: "composite"
+  steps:
+    - name: Get AWS credentials for sccache bucket
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+        aws-region: us-east-2 
+        role-duration-seconds: 43200 # 12 hours)
+    - name: Set environment variables
+      run: |
+        echo "SCCACHE_BUCKET=rapids-sccache-east" >> $GITHUB_ENV
+        echo "SCCACHE_REGION=us-east-2" >> $GITHUB_ENV
+        echo "SCCACHE_IDLE_TIMEOUT=32768" >> $GITHUB_ENV
+        echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV
+        echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV
diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 473d29ed368..9ac84636ed2 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -35,7 +35,7 @@ jobs:
         fetch-depth: 0
         persist-credentials: false
     - name: Configure credentials and environment variables for sccache
-      uses: ./.github/workflows/configure_cccl_sccache.yml
+      uses: ./.github/actions/configure_cccl_sccache
     - name: Run build script
       run: |
         chmod +x ${{ inputs.build_script }}
@@ -65,7 +65,7 @@ jobs:
         fetch-depth: 0
         persist-credentials: false
     - name: Configure credentials and environment variables for sccache
-      uses: ./.github/workflows/configure_cccl_sccache.yml
+      uses: ./.github/actions/configure_cccl_sccache
     - name: Run test script
       run: |
         chmod +x ${{ inputs.test_script }}
diff --git a/.github/workflows/configure_cccl_sccache.yml b/.github/workflows/configure_cccl_sccache.yml
deleted file mode 100644
index 2e26f1a498a..00000000000
--- a/.github/workflows/configure_cccl_sccache.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: Set up AWS credentials and environment variables for sccache
-
-on:
-  workflow_call:
-    steps:
-      - name: Get AWS credentials for sccache bucket
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-          aws-region: us-east-2 
-          role-duration-seconds: 43200 # 12 hours)
-      - name: Set environment variables
-        run: |
-          echo "SCCACHE_BUCKET=rapids-sccache-east" >> $GITHUB_ENV
-          echo "SCCACHE_REGION=us-east-2" >> $GITHUB_ENV
-          echo "SCCACHE_IDLE_TIMEOUT=32768" >> $GITHUB_ENV
-          echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV
-          echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV

From 7aedd4a09dd3000bfccb6baa9c7687ceda6dcf97 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 11:14:00 -0500
Subject: [PATCH 136/395] Add shell property to composite action.

---
 .github/actions/configure_cccl_sccache/action.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/configure_cccl_sccache/action.yml b/.github/actions/configure_cccl_sccache/action.yml
index b47de57ba9d..6aac3cdcbd6 100644
--- a/.github/actions/configure_cccl_sccache/action.yml
+++ b/.github/actions/configure_cccl_sccache/action.yml
@@ -1,5 +1,6 @@
 name: Set up AWS credentials and environment variables for sccache
 description: "Set up AWS credentials and environment variables for sccache"
+shell: bash
 runs:
   using: "composite"
   steps:

From 5df4b2d5175b9afcfec20c1ad36ae80eb840f4b5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 11:18:09 -0500
Subject: [PATCH 137/395] Move shell property.

---
 .github/actions/configure_cccl_sccache/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/configure_cccl_sccache/action.yml b/.github/actions/configure_cccl_sccache/action.yml
index 6aac3cdcbd6..17a49fe7ff1 100644
--- a/.github/actions/configure_cccl_sccache/action.yml
+++ b/.github/actions/configure_cccl_sccache/action.yml
@@ -1,6 +1,5 @@
 name: Set up AWS credentials and environment variables for sccache
 description: "Set up AWS credentials and environment variables for sccache"
-shell: bash
 runs:
   using: "composite"
   steps:
@@ -17,3 +16,4 @@ runs:
         echo "SCCACHE_IDLE_TIMEOUT=32768" >> $GITHUB_ENV
         echo "SCCACHE_S3_USE_SSL=true" >> $GITHUB_ENV
         echo "SCCACHE_S3_NO_CREDENTIALS=false" >> $GITHUB_ENV
+      shell: bash

From 5a26d0c29840692f2be58e835a0b11cdeca43373 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 18:30:48 -0500
Subject: [PATCH 138/395] Add GPUs to matrix and compute separate build/test
 matrices.

---
 .github/workflows/base.yml | 11 +++++++----
 .github/workflows/test.yml | 19 +++++++++++++------
 ci/matrix.yaml             |  4 ++++
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 9ac84636ed2..a3eeefddf89 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -3,7 +3,10 @@ name: Base CCCL workflow
 on:
   workflow_call:
     inputs:
-      matrix: # The matrix to use for this job
+      build_matrix: # The matrix to use for this job
+        type: string
+        required: true
+      test_matrix: # The matrix to use for this job
         type: string
         required: true
       build_script: # The script to run for the build step
@@ -20,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJSON(inputs.matrix) }}
+      matrix: ${{ fromJSON(inputs.build_matrix) }}
     runs-on: linux-${{ matrix.cpu }}-cpu16
     container: # Fetch the container image to use for the job
       options: -u root
@@ -48,8 +51,8 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJSON(inputs.matrix) }}
-    runs-on: linux-${{matrix.cpu}}-gpu-v100-latest-1
+      matrix: ${{ fromJSON(inputs.test_matrix) }}
+    runs-on: linux-${{matrix.cpu}}-gpu-${{matrix.gpu}}-latest-1
     container: # Fetch the container image to use for the job
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f05f45471f..360b712c47e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,7 +19,8 @@ jobs:
   compute-matrix:
     runs-on: ubuntu-latest
     outputs:
-      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
+      BUILD_MATRIX: ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
+      TEST_MATRIX:  ${{ steps.compute-matrix.outputs.TEST_MATRIX }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -27,14 +28,18 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ] | {include: .}')
-          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ] | {include: .}')
+          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
+          TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
+          echo "TEST_MATRIX=$TEST_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   thrust:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
-      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 
@@ -42,7 +47,8 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
-      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
       build_script: "./ci/build_libcudacxx.sh"
       test_script: "./ci/test_libcudacxx.sh"
 
@@ -50,7 +56,8 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/base.yml
     with:
-      matrix: ${{ needs.compute-matrix.outputs.MATRIX }}
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
       build_script: "./ci/build_cub.sh"
       test_script: "./ci/test_cub.sh"
 
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 35743a645fb..0ce518bfa8a 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -6,6 +6,10 @@ std:
   - 17
   - 20
 
+gpus:
+  - 'a100'
+  - 'v100'
+
 include:
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}

From efd556c9d9c24497831ac3056047e07aa427e022 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 19:41:06 -0500
Subject: [PATCH 139/395] Add GPU name to test job name.

---
 .github/workflows/base.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index a3eeefddf89..3a414875d29 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -47,7 +47,7 @@ jobs:
   test:
     needs: build
     if: github.repository == 'NVIDIA/cccl'
-    name: Test/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}})
+    name: Test/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.gpu}}/${{matrix.cpu}})
     strategy:
       fail-fast: false
       max-parallel: 8

From 596fc92b28dff25d590a15fe96ef012032cf732e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 20:15:34 -0500
Subject: [PATCH 140/395] Update matrix to specify C++ dialect per environment.

Updated compute-matrix logic to compute the cartesian product per environment
with the C++ dialect specified in std.
---
 .github/workflows/test.yml |  2 +-
 ci/matrix.yaml             | 42 ++++++++++++++++----------------------
 2 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 360b712c47e..5293a6e0642 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include, .std] | [ combinations | .[0] + {std: .[1]} ] | {include: .}')
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
           GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
           TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 0ce518bfa8a..57682d7d9a7 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -1,33 +1,27 @@
 #gcc
 
-std: 
-  - 11
-  - 14
-  - 17
-  - 20
-
 gpus:
   - 'a100'
   - 'v100'
 
 include:
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
 
  #clang
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}

From 36626d280b911bd65f5b1acbca8a4d898249f6e9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 20:22:29 -0500
Subject: [PATCH 141/395] Add comments to matrix file.

---
 ci/matrix.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 57682d7d9a7..025f5b37894 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -1,9 +1,23 @@
 #gcc
 
+
+# The GPUs to test on
+# Note: This assumes that the appropriate gpu_build_archs are set to include building for the GPUs listed here
 gpus:
   - 'a100'
   - 'v100'
 
+# Each environment below will generate a unique build/test job
+# cuda: The CUDA Toolkit version
+# ubuntu: The Ubuntu version
+# cpu: The CPU architecture
+# compiler: The compiler to use
+#   name: The compiler name
+#   version: The compiler version
+#   exe: The unverionsed compiler binary name
+# gpu_build_archs: The GPU architectures to build for
+# std: The C++ standards to build for
+#    This field is unique as it will generate an independent build/test job for each value
 include:
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}

From b4513458b22d3571690615b0002658e119b77a0f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 26 May 2023 20:35:28 -0500
Subject: [PATCH 142/395] Remove unnecessar chmod before running scripts.

---
 .github/workflows/base.yml | 2 --
 ci/matrix.yaml             | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 3a414875d29..7848bfca935 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -41,7 +41,6 @@ jobs:
       uses: ./.github/actions/configure_cccl_sccache
     - name: Run build script
       run: |
-        chmod +x ${{ inputs.build_script }}
         ./${{ inputs.build_script }}
         sccache -s
   test:
@@ -71,6 +70,5 @@ jobs:
       uses: ./.github/actions/configure_cccl_sccache
     - name: Run test script
       run: |
-        chmod +x ${{ inputs.test_script }}
         ./${{ inputs.test_script }}
         sccache -s
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 025f5b37894..0e74c5791bc 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -8,6 +8,7 @@ gpus:
   - 'v100'
 
 # Each environment below will generate a unique build/test job
+# See the "compute-matrix" job in the workflow for how this is parsed and used
 # cuda: The CUDA Toolkit version
 # ubuntu: The Ubuntu version
 # cpu: The CPU architecture

From d4e7fa414e965b23f6ad9aac389565d04e87f075 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Sat, 27 May 2023 08:51:07 -0500
Subject: [PATCH 143/395] Rename files with more descriptive names.

---
 .github/workflows/{base.yml => build_and_test.yml} |  0
 .github/workflows/{test.yml => pr.yml}             | 11 ++++++-----
 2 files changed, 6 insertions(+), 5 deletions(-)
 rename .github/workflows/{base.yml => build_and_test.yml} (100%)
 rename .github/workflows/{test.yml => pr.yml} (92%)

diff --git a/.github/workflows/base.yml b/.github/workflows/build_and_test.yml
similarity index 100%
rename from .github/workflows/base.yml
rename to .github/workflows/build_and_test.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/pr.yml
similarity index 92%
rename from .github/workflows/test.yml
rename to .github/workflows/pr.yml
index 5293a6e0642..0d04abba3e6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/pr.yml
@@ -1,5 +1,5 @@
-name: CI (GPU)
-
+# This is the main workflow that runs on every PR and push to main
+name: pr
 
 defaults:
   run:
@@ -11,6 +11,7 @@ on:
       - main
       - "pull-request/[0-9]+"
 
+# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
 concurrency:
   group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
@@ -36,7 +37,7 @@ jobs:
 
   thrust:
     needs: compute-matrix
-    uses: ./.github/workflows/base.yml
+    uses: ./.github/workflows/build_and_test.yml
     with:
       build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
       test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
@@ -45,7 +46,7 @@ jobs:
 
   libcudacxx:
     needs: compute-matrix
-    uses: ./.github/workflows/base.yml
+    uses: ./.github/workflows/build_and_test.yml
     with:
       build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
       test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
@@ -54,7 +55,7 @@ jobs:
 
   cub:
     needs: compute-matrix
-    uses: ./.github/workflows/base.yml
+    uses: ./.github/workflows/build_and_test.yml
     with:
       build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
       test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}

From b9dad1164ea8b931fbb7e7c4014dc6f015412d5e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:36:58 -0500
Subject: [PATCH 144/395] Remove stale comments.

---
 .github/workflows/build_and_test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 7848bfca935..ec917fbf092 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -3,13 +3,13 @@ name: Base CCCL workflow
 on:
   workflow_call:
     inputs:
-      build_matrix: # The matrix to use for this job
+      build_matrix: 
         type: string
         required: true
-      test_matrix: # The matrix to use for this job
+      test_matrix: 
         type: string
         required: true
-      build_script: # The script to run for the build step
+      build_script:
         type: string
         required: true
       test_script: 

From 082492e775cb1f5dc9d20bb2e47375395db0697d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:37:32 -0500
Subject: [PATCH 145/395] Disable current workflow and test split workflow.

---
 .github/workflows/per_cuda.yml          | 16 +++++++++
 .github/workflows/pr.yml                | 10 +++---
 .github/workflows/test_new_workflow.yml | 46 +++++++++++++++++++++++++
 ci/get_groups.sh                        | 16 +++++++++
 ci/group_by_field.jq                    |  4 +++
 5 files changed, 87 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/per_cuda.yml
 create mode 100644 .github/workflows/test_new_workflow.yml
 create mode 100755 ci/get_groups.sh
 create mode 100644 ci/group_by_field.jq

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
new file mode 100644
index 00000000000..0e99ec2bec5
--- /dev/null
+++ b/.github/workflows/per_cuda.yml
@@ -0,0 +1,16 @@
+name: Per CUDA version
+
+on:
+  workflow_call:
+    inputs:
+      per_cuda_matrix: {type: string, required: true}
+
+test-repo-per-cuda-ver:
+  uses: ./.github/workflows/test-repo-per-compiler.yml
+  strategy:
+    matrix: ${{ fromJSON(inputs.cuda_vers) }}
+  with:
+    repo: ${{ inputs.repo }}
+    cxx_vers: ${{ inputs.cxx_vers }}
+    cuda_ver: ${{ matrix.cuda_ver }}
+    compilers: ${{ inputs.compilers }}
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0d04abba3e6..9013a9fc8c3 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -5,11 +5,11 @@ defaults:
   run:
     shell: bash
 
-on:
-  push:
-    branches:
-      - main
-      - "pull-request/[0-9]+"
+#on:
+#  push:
+#    branches:
+#      - main
+#      - "pull-request/[0-9]+"
 
 # Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
 concurrency:
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
new file mode 100644
index 00000000000..f394185a710
--- /dev/null
+++ b/.github/workflows/test_new_workflow.yml
@@ -0,0 +1,46 @@
+
+# This is the main workflow that runs on every PR and push to main
+name: pr_split
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  push:
+    branches:
+      - main
+      - "pull-request/[0-9]+"
+
+# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
+concurrency:
+  group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  compute-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      BUILD_MATRIX:  ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
+      PER_CUDA_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_MATRIX }}
+      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          cat ./ci/matrix.yaml
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
+          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_MATRIX=./ci/get_groups.sh $BUILD_MATRIX cuda $GITHUB_OUTPUT
+          CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq 'keys')
+          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          
+  thrust:
+    needs: compute-matrix
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }}  }}
+    with:
+      per_cuda_matrix: ${{ needs.compute-matrix.outputs.PER_CUDA_MATRIX }}
\ No newline at end of file
diff --git a/ci/get_groups.sh b/ci/get_groups.sh
new file mode 100755
index 00000000000..e41152d5e08
--- /dev/null
+++ b/ci/get_groups.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Check if input JSON, field, and file arguments are provided
+if [ "$#" -ne 3 ]; then
+  echo "Usage: ./get_groups.sh <input_json> <field> <output_file>"
+  exit 1
+fi
+
+# Assign command-line arguments to variables
+input_json="$1"
+field="$2"
+output_file="$3"
+
+output=$(echo $input_json | jq -c --arg field "$field" 'include "group_by_field"; .include | group_by_field($field)')
+echo $output | tee -a "$output_file"
+
diff --git a/ci/group_by_field.jq b/ci/group_by_field.jq
new file mode 100644
index 00000000000..d074a0a655c
--- /dev/null
+++ b/ci/group_by_field.jq
@@ -0,0 +1,4 @@
+# Groups an array of objects by a field and returns an object with the field value as key and the array of objects as value
+# Handles nested fields (e.g. "compiler.name")
+def group_by_field($field):
+    group_by(getpath($field | split("."))) | map( {(.[0] | getpath($field | split("."))): .}) | add;

From 75a9162a92a95c91a04a884646087ed5fbbe7905 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:39:53 -0500
Subject: [PATCH 146/395] Add job to per_cuda workflow.

---
 .github/workflows/per_cuda.yml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 0e99ec2bec5..f12c9a6afaa 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -5,12 +5,10 @@ on:
     inputs:
       per_cuda_matrix: {type: string, required: true}
 
-test-repo-per-cuda-ver:
-  uses: ./.github/workflows/test-repo-per-compiler.yml
-  strategy:
-    matrix: ${{ fromJSON(inputs.cuda_vers) }}
-  with:
-    repo: ${{ inputs.repo }}
-    cxx_vers: ${{ inputs.cxx_vers }}
-    cuda_ver: ${{ matrix.cuda_ver }}
-    compilers: ${{ inputs.compilers }}
\ No newline at end of file
+
+jobs:
+  per_cuda:
+    run: |
+      echo "per_cuda_matrix: ${{ inputs.per_cuda_matrix }}"
+
+

From 7529df6f5241d01371d3274921dd3cd957f0c119 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:40:58 -0500
Subject: [PATCH 147/395] Add step to job.

---
 .github/workflows/per_cuda.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index f12c9a6afaa..bd8e5257304 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,7 +8,9 @@ on:
 
 jobs:
   per_cuda:
-    run: |
-      echo "per_cuda_matrix: ${{ inputs.per_cuda_matrix }}"
+    steps:
+      - name: test
+        run: |
+          echo "per_cuda_matrix: ${{ inputs.per_cuda_matrix }}"
 
 

From 17b58ec252433fb361096e3dc400a546f23a3c4a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:42:04 -0500
Subject: [PATCH 148/395] Resolve error about missing "on" by triggering only
 on push to main.

---
 .github/workflows/pr.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 9013a9fc8c3..17ba858ee09 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -5,10 +5,10 @@ defaults:
   run:
     shell: bash
 
-#on:
-#  push:
-#    branches:
-#      - main
+on:
+  push:
+    branches:
+      - main
 #      - "pull-request/[0-9]+"
 
 # Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.

From d14fca534938a67ea2fd5200f1fae47599ce8b33 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:42:58 -0500
Subject: [PATCH 149/395] Add missing runs-on to test job.

---
 .github/workflows/per_cuda.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index bd8e5257304..6494bbcf249 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   per_cuda:
+    runs-on: ubuntu-latest
     steps:
       - name: test
         run: |

From d4d50ba19529f21d7cbce24aa32cf1b4a3b89b58 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:45:11 -0500
Subject: [PATCH 150/395] Wrap per_cuda_matrix initialization.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index f394185a710..497ad5e9b4d 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -33,7 +33,7 @@ jobs:
           cat ./ci/matrix.yaml
           BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_MATRIX=./ci/get_groups.sh $BUILD_MATRIX cuda $GITHUB_OUTPUT
+          PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda "$GITHUB_OUTPUT")
           CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq 'keys')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
           

From ec4f8e72805f3d23d139494cd25ed642b6dce247 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:47:08 -0500
Subject: [PATCH 151/395] Update jq command to search locally for script.

---
 ci/get_groups.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/get_groups.sh b/ci/get_groups.sh
index e41152d5e08..a1ccbcae3c0 100755
--- a/ci/get_groups.sh
+++ b/ci/get_groups.sh
@@ -11,6 +11,6 @@ input_json="$1"
 field="$2"
 output_file="$3"
 
-output=$(echo $input_json | jq -c --arg field "$field" 'include "group_by_field"; .include | group_by_field($field)')
+output=$(echo $input_json | jq -L . -c --arg field "$field" 'include "group_by_field"; .include | group_by_field($field)')
 echo $output | tee -a "$output_file"
 

From a30818bc6f7c72b8c4271064d162f0526ff98df3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:51:31 -0500
Subject: [PATCH 152/395] Update get_groups to ensure it executes from its
 directory.

---
 ci/get_groups.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/get_groups.sh b/ci/get_groups.sh
index a1ccbcae3c0..afcbd6dd4bd 100755
--- a/ci/get_groups.sh
+++ b/ci/get_groups.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Ensure that the script is being executed from its directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
 # Check if input JSON, field, and file arguments are provided
 if [ "$#" -ne 3 ]; then
   echo "Usage: ./get_groups.sh <input_json> <field> <output_file>"

From f81a015ef182d767396272515662b0b8705a8d33 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 07:59:30 -0500
Subject: [PATCH 153/395] Update how output is returned from get_groups

---
 .github/workflows/test_new_workflow.yml | 3 ++-
 ci/get_groups.sh                        | 9 ++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 497ad5e9b4d..402558f50f4 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -33,7 +33,8 @@ jobs:
           cat ./ci/matrix.yaml
           BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda "$GITHUB_OUTPUT")
+          PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda)
+          echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT
           CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq 'keys')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
           
diff --git a/ci/get_groups.sh b/ci/get_groups.sh
index afcbd6dd4bd..275759017df 100755
--- a/ci/get_groups.sh
+++ b/ci/get_groups.sh
@@ -3,17 +3,16 @@
 # Ensure that the script is being executed from its directory
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
-# Check if input JSON, field, and file arguments are provided
-if [ "$#" -ne 3 ]; then
-  echo "Usage: ./get_groups.sh <input_json> <field> <output_file>"
+# Check if input JSON and field arguments are provided
+if [ "$#" -ne 2 ]; then
+  echo "Usage: ./get_groups.sh <input_json> <field>"
   exit 1
 fi
 
 # Assign command-line arguments to variables
 input_json="$1"
 field="$2"
-output_file="$3"
 
 output=$(echo $input_json | jq -L . -c --arg field "$field" 'include "group_by_field"; .include | group_by_field($field)')
-echo $output | tee -a "$output_file"
+echo $output 
 

From 89418bdaeb168cf66437e0bf94ffcc48647ecd52 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:00:34 -0500
Subject: [PATCH 154/395] Add missing quote

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 402558f50f4..246516d1141 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -34,7 +34,7 @@ jobs:
           BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
           PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda)
-          echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT
+          echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT"
           CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq 'keys')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
           

From 3600cb3cf9783ba1750f91e148ab863bc719455d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:06:33 -0500
Subject: [PATCH 155/395] Use jq -c for keys.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 246516d1141..6420b3d5d08 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -35,7 +35,7 @@ jobs:
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
           PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda)
           echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq 'keys')
+          CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq -c 'keys')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
           
   thrust:

From 6d48f3ed08cd39c4b80997c31961d07c9ad105bd Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:08:28 -0500
Subject: [PATCH 156/395] Remove extraneous braces.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 6420b3d5d08..4af00c0bb3e 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -42,6 +42,6 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }}  }}
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
       per_cuda_matrix: ${{ needs.compute-matrix.outputs.PER_CUDA_MATRIX }}
\ No newline at end of file

From ba038ff47905610c7f033a9d9635b5555c292bbd Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:13:15 -0500
Subject: [PATCH 157/395] Update per_cuda matrix to use a cuda_version in the
 matrix.

---
 .github/workflows/test_new_workflow.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 4af00c0bb3e..bde1e7179e4 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -42,6 +42,7 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:
-      matrix: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
       per_cuda_matrix: ${{ needs.compute-matrix.outputs.PER_CUDA_MATRIX }}
\ No newline at end of file

From cf9c8c4ec88cb935709fc8228edf8b787be7a6f9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:30:49 -0500
Subject: [PATCH 158/395] Attempt to pass per CUDA matrix to nested workflow.

---
 .github/workflows/per_cuda.yml          | 2 ++
 .github/workflows/test_new_workflow.yml | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 6494bbcf249..5fb6404bc5d 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,6 +9,8 @@ on:
 jobs:
   per_cuda:
     runs-on: ubuntu-latest
+        strategy:
+      matrix: ${{ fromJSON(inputs.per_cuda_matrix) }}
     steps:
       - name: test
         run: |
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index bde1e7179e4..9f6ec3d6bdb 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -45,4 +45,4 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
-      per_cuda_matrix: ${{ needs.compute-matrix.outputs.PER_CUDA_MATRIX }}
\ No newline at end of file
+      per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
\ No newline at end of file

From 794c1a58fe60009d4ec2be52bd9a90015b4d2222 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:31:27 -0500
Subject: [PATCH 159/395] Fix syntax

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 5fb6404bc5d..d8f5e4b8c8d 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   per_cuda:
     runs-on: ubuntu-latest
-        strategy:
+    strategy:
       matrix: ${{ fromJSON(inputs.per_cuda_matrix) }}
     steps:
       - name: test

From ff51fe83e525d3695c0a34cf1acd28d69566a480 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:36:45 -0500
Subject: [PATCH 160/395] Update per_cuda to pass the per_cuda_matrix to
 include:

---
 .github/workflows/per_cuda.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index d8f5e4b8c8d..ef657ba1148 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -10,7 +10,8 @@ jobs:
   per_cuda:
     runs-on: ubuntu-latest
     strategy:
-      matrix: ${{ fromJSON(inputs.per_cuda_matrix) }}
+      matrix: 
+        include: ${{ fromJSON(inputs.per_cuda_matrix) }}
     steps:
       - name: test
         run: |

From 942429dc5b778c37107660b2f0f29ab30ab0ba47 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:44:25 -0500
Subject: [PATCH 161/395] Test if I can access field fromJSON

---
 .github/workflows/test_new_workflow.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 9f6ec3d6bdb..4f29f84bf46 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -40,9 +40,14 @@ jobs:
           
   thrust:
     needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
+    #uses: ./.github/workflows/per_cuda.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-    with:
-      per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
\ No newline at end of file
+    steps:
+      - name: test
+        run: |
+          echo "cuda_version: ${{ matrix.cuda_version }}"
+          echo "PER_CUDA_MATRIX: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}"
+    #with:
+    #  per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
\ No newline at end of file

From 4d9fe866d9926024e022d1861e15e36925f8639b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:44:55 -0500
Subject: [PATCH 162/395] Add missing runs-on

---
 .github/workflows/test_new_workflow.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 4f29f84bf46..c7b636b3ce5 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -44,6 +44,7 @@ jobs:
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+    runs-on: ubuntu-latest
     steps:
       - name: test
         run: |

From 89d9b2d943f243e3b8287d5e9e14f0c646324b23 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 08:47:46 -0500
Subject: [PATCH 163/395] Wrap with toJSON to get it to print.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index c7b636b3ce5..94a27c08bfb 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -49,6 +49,6 @@ jobs:
       - name: test
         run: |
           echo "cuda_version: ${{ matrix.cuda_version }}"
-          echo "PER_CUDA_MATRIX: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}"
+          echo "PER_CUDA_MATRIX: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}"
     #with:
     #  per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
\ No newline at end of file

From 8ae3755b3e75c57a56b5f1c6c0f460d7c6e5d7f4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:31:53 -0500
Subject: [PATCH 164/395] Try passing the per_cuda_matrix to the per_cuda
 workflow again.

---
 .github/workflows/per_cuda.yml          |  3 ---
 .github/workflows/test_new_workflow.yml | 18 +++++++++---------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index ef657ba1148..6494bbcf249 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,9 +9,6 @@ on:
 jobs:
   per_cuda:
     runs-on: ubuntu-latest
-    strategy:
-      matrix: 
-        include: ${{ fromJSON(inputs.per_cuda_matrix) }}
     steps:
       - name: test
         run: |
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 94a27c08bfb..daec67d26fa 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -40,15 +40,15 @@ jobs:
           
   thrust:
     needs: compute-matrix
-    #uses: ./.github/workflows/per_cuda.yml
+    uses: ./.github/workflows/per_cuda.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-    runs-on: ubuntu-latest
-    steps:
-      - name: test
-        run: |
-          echo "cuda_version: ${{ matrix.cuda_version }}"
-          echo "PER_CUDA_MATRIX: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}"
-    #with:
-    #  per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
\ No newline at end of file
+    with:
+      per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
+    #runs-on: ubuntu-latest
+    #steps:
+    #  - name: test
+    #    run: |
+    #      echo "cuda_version: ${{ matrix.cuda_version }}"
+    #      echo "PER_CUDA_MATRIX: ${{ fromJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}"
\ No newline at end of file

From 11b3423a2db48376c96d9ce4f24bcff3cd00ce5e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:33:38 -0500
Subject: [PATCH 165/395] Wrap with toJSON.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index daec67d26fa..b513bc016f7 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -45,7 +45,7 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
-      per_cuda_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version] }}
+      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
     #runs-on: ubuntu-latest
     #steps:
     #  - name: test

From d5f1d522216f14a88432edaa90f1c7a051fa27ab Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:36:02 -0500
Subject: [PATCH 166/395] Make per_cuda a matrix job on the provided matrix.

---
 .github/workflows/per_cuda.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 6494bbcf249..8178ccd1967 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,9 +9,12 @@ on:
 jobs:
   per_cuda:
     runs-on: ubuntu-latest
+    strategy:
+      matrix: 
+        include: ${{ fromJSON(inputs.per_cuda_matrix) }}
     steps:
       - name: test
         run: |
-          echo "per_cuda_matrix: ${{ inputs.per_cuda_matrix }}"
+          echo "${{ matrix.compiler.name}}" 
 
 

From 7ce3f0ba2e605b81a9dc00355bc225d6d7405b9c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:38:43 -0500
Subject: [PATCH 167/395] Remove commented code.

---
 .github/workflows/test_new_workflow.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index b513bc016f7..996ea687ea6 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -45,10 +45,4 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
-      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
-    #runs-on: ubuntu-latest
-    #steps:
-    #  - name: test
-    #    run: |
-    #      echo "cuda_version: ${{ matrix.cuda_version }}"
-    #      echo "PER_CUDA_MATRIX: ${{ fromJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}"
\ No newline at end of file
+      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
\ No newline at end of file

From 0af43bb99fce410d92bc5dd3bd0f90c3b430cf4b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:45:03 -0500
Subject: [PATCH 168/395] Add cuda version to per_cuda job.

---
 .github/workflows/per_cuda.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 8178ccd1967..b7db985b086 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   per_cuda:
+    name: CUDA ${{ matrix.cuda_version }}
     runs-on: ubuntu-latest
     strategy:
       matrix: 

From 9a1a6edbe87f3af0c1e982ae57747937ac9c2f1b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:50:22 -0500
Subject: [PATCH 169/395] Add per_compiler workflow.

---
 .github/workflows/per_compiler.yml      | 18 +++++++++++++++
 .github/workflows/per_cuda.yml          | 30 ++++++++++++++++++-------
 .github/workflows/test_new_workflow.yml |  3 ++-
 3 files changed, 42 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/per_compiler.yml

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
new file mode 100644
index 00000000000..215afad3328
--- /dev/null
+++ b/.github/workflows/per_compiler.yml
@@ -0,0 +1,18 @@
+name: Per compiler
+
+on:
+  workflow_call:
+    inputs:
+      per_compiler_matrix: {type: string, required: true}
+      compiler_name: {type: string, required: true}
+
+jobs:
+  per_compiler:
+    runs-on: ubuntu-latest
+    steps:
+      - name: test
+        run: |
+          echo "${{ inputs.compiler_name }}"
+          echo "${{ fromJSON(inputs.per_compiler_matrix) }}" 
+
+
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index b7db985b086..949a808d10d 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -4,18 +4,32 @@ on:
   workflow_call:
     inputs:
       per_cuda_matrix: {type: string, required: true}
-
+      cuda_version: {type: string, required: true}
 
 jobs:
-  per_cuda:
-    name: CUDA ${{ matrix.cuda_version }}
+  compute_per_compiler_matrix:
     runs-on: ubuntu-latest
-    strategy:
-      matrix: 
-        include: ${{ fromJSON(inputs.per_cuda_matrix) }}
+    outputs:
+      PER_COMPILER_MATRIX: ${{ steps.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX }}
+      COMPILERS: ${{ steps.compute_per_compiler_matrix.outputs.COMPILERS }}
     steps:
-      - name: test
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
         run: |
-          echo "${{ matrix.compiler.name}}" 
+          PER_COMPILER_MATRIX=$(./ci/get_groups.sh ${{inputs.per_compiler_matrix}} compiler)
+          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          COMPILERS=$(echo $PER_COMPILER_MATRIX | jq -c 'keys')
+          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
+
+  per_cuda:
+    needs: compute_per_compiler_matrix
+    name: CUDA ${{ inputs.cuda_version }}
+    strategy:
+      matrix: 
+        compiler: ${{ fromJSON(needs.compute_per_compiler_matrix.COMPILERS) }}
+    uses: ./.github/workflows/per_compiler.yml
+    with:
+      per_compiler_matrix: ${{ toJSON(fromJSON(needs.compute_per_compiler_matrix.PER_COMPILER_MATRIX)[matrix.compiler]) }}
 
 
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 996ea687ea6..108ad6da1f3 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -45,4 +45,5 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
-      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
\ No newline at end of file
+      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
+      cuda_version: ${{ matrix.cuda_version }}
\ No newline at end of file

From 958fa09eaa6445ea2e12a8eea1c4f12c7a4ef20f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:51:08 -0500
Subject: [PATCH 170/395] Add missing compiler_name input.

---
 .github/workflows/per_cuda.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 949a808d10d..94af8785858 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -31,5 +31,6 @@ jobs:
     uses: ./.github/workflows/per_compiler.yml
     with:
       per_compiler_matrix: ${{ toJSON(fromJSON(needs.compute_per_compiler_matrix.PER_COMPILER_MATRIX)[matrix.compiler]) }}
+      compiler_name: ${{ matrix.compiler }}
 
 

From 606fdd83678004675b49b5e591057dab9c166eb3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:54:14 -0500
Subject: [PATCH 171/395] Don't wrap build matrix in "include{}" to avoid hard
 coding into the script.

---
 .github/workflows/test_new_workflow.yml | 2 +-
 ci/get_groups.sh                        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 108ad6da1f3..4ee78bb54d8 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -31,7 +31,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)]' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
           PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda)
           echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT"
diff --git a/ci/get_groups.sh b/ci/get_groups.sh
index 275759017df..e7c0c256c51 100755
--- a/ci/get_groups.sh
+++ b/ci/get_groups.sh
@@ -13,6 +13,6 @@ fi
 input_json="$1"
 field="$2"
 
-output=$(echo $input_json | jq -L . -c --arg field "$field" 'include "group_by_field"; .include | group_by_field($field)')
+output=$(echo $input_json | jq -L . -c --arg field "$field" 'include "group_by_field"; group_by_field($field)')
 echo $output 
 

From fba52e4039059b6a50c63ac9df3f9bffd750fbc1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:56:23 -0500
Subject: [PATCH 172/395] Correctly pass per_cuda_matrix into get_groups

---
 .github/workflows/per_cuda.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 94af8785858..e32899d18e4 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -17,12 +17,14 @@ jobs:
         uses: actions/checkout@v3
       - name: Compute matrix
         run: |
-          PER_COMPILER_MATRIX=$(./ci/get_groups.sh ${{inputs.per_compiler_matrix}} compiler)
+          echo ${{ inputs.per_cuda_matrix }} 
+          PER_COMPILER_MATRIX=$(./ci/get_groups.sh ${{inputs.per_cuda_matrix}} compiler.name)
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
           COMPILERS=$(echo $PER_COMPILER_MATRIX | jq -c 'keys')
           echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
 
   per_cuda:
+    name: ${{ matrix.compiler }}
     needs: compute_per_compiler_matrix
     name: CUDA ${{ inputs.cuda_version }}
     strategy:

From 47dfc055a406a228372c5f95c5ee6688325c0b0d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:57:11 -0500
Subject: [PATCH 173/395] Extra name field.

---
 .github/workflows/per_cuda.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index e32899d18e4..6e8f2b3dd02 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -26,7 +26,6 @@ jobs:
   per_cuda:
     name: ${{ matrix.compiler }}
     needs: compute_per_compiler_matrix
-    name: CUDA ${{ inputs.cuda_version }}
     strategy:
       matrix: 
         compiler: ${{ fromJSON(needs.compute_per_compiler_matrix.COMPILERS) }}

From 071a3c3ebc70dee3d35ee1a841e0e17106652a7d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 09:59:37 -0500
Subject: [PATCH 174/395] Wrap the inputs.per_cuda_matrix in quotes.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 6e8f2b3dd02..bfdae1d8ae5 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Compute matrix
         run: |
-          echo ${{ inputs.per_cuda_matrix }} 
+          echo "${{ inputs.per_cuda_matrix }}"
           PER_COMPILER_MATRIX=$(./ci/get_groups.sh ${{inputs.per_cuda_matrix}} compiler.name)
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
           COMPILERS=$(echo $PER_COMPILER_MATRIX | jq -c 'keys')

From 64623ffd2e90bf1e2949a0db37ce0f4234701e8c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:06:08 -0500
Subject: [PATCH 175/395] Store input to environment variable.

---
 .github/workflows/per_cuda.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index bfdae1d8ae5..5c2348606a3 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -17,8 +17,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Compute matrix
         run: |
-          echo "${{ inputs.per_cuda_matrix }}"
-          PER_COMPILER_MATRIX=$(./ci/get_groups.sh ${{inputs.per_cuda_matrix}} compiler.name)
+          export PER_CUDA_MATRIX="${{ fromJSON(inputs.per_cuda_matrix) }}"
+          echo $PER_CUDA_MATRIX
+          PER_COMPILER_MATRIX=$(./ci/get_groups.sh $PER_CUDA_MATRIX compiler.name)
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
           COMPILERS=$(echo $PER_COMPILER_MATRIX | jq -c 'keys')
           echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"

From c9e2e99b86c6a12a838702551a1afb6f4746b26a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:07:58 -0500
Subject: [PATCH 176/395] Don't use fromJSON when assigning to variable.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 5c2348606a3..e68fbe2dcbd 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Compute matrix
         run: |
-          export PER_CUDA_MATRIX="${{ fromJSON(inputs.per_cuda_matrix) }}"
+          export PER_CUDA_MATRIX="${{ inputs.per_cuda_matrix }}"
           echo $PER_CUDA_MATRIX
           PER_COMPILER_MATRIX=$(./ci/get_groups.sh $PER_CUDA_MATRIX compiler.name)
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"

From 8d890d006d456668aec8f2666c6df97b331934c1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:21:38 -0500
Subject: [PATCH 177/395] Use new approach where entire matrix is passed
 through and later filtered.

---
 .github/workflows/per_compiler.yml      |  6 +++---
 .github/workflows/per_cuda.yml          | 25 +++++--------------------
 .github/workflows/test_new_workflow.yml | 14 ++++++++------
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 215afad3328..91860d0355a 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -3,7 +3,8 @@ name: Per compiler
 on:
   workflow_call:
     inputs:
-      per_compiler_matrix: {type: string, required: true}
+      cuda_version: {type: string, required: true}
+      full_matrix: {type: string, required: true}
       compiler_name: {type: string, required: true}
 
 jobs:
@@ -12,7 +13,6 @@ jobs:
     steps:
       - name: test
         run: |
-          echo "${{ inputs.compiler_name }}"
-          echo "${{ fromJSON(inputs.per_compiler_matrix) }}" 
+          echo "CUDA: ${{inputs.cuda_version}} Compiler: ${{ inputs.compiler_name }}"
 
 
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index e68fbe2dcbd..080311d3be1 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -3,36 +3,21 @@ name: Per CUDA version
 on:
   workflow_call:
     inputs:
-      per_cuda_matrix: {type: string, required: true}
       cuda_version: {type: string, required: true}
+      full_matrix: {type: string, required: true}
+      compilers: {type: string, required: true}
 
 jobs:
-  compute_per_compiler_matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      PER_COMPILER_MATRIX: ${{ steps.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX }}
-      COMPILERS: ${{ steps.compute_per_compiler_matrix.outputs.COMPILERS }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Compute matrix
-        run: |
-          export PER_CUDA_MATRIX="${{ inputs.per_cuda_matrix }}"
-          echo $PER_CUDA_MATRIX
-          PER_COMPILER_MATRIX=$(./ci/get_groups.sh $PER_CUDA_MATRIX compiler.name)
-          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          COMPILERS=$(echo $PER_COMPILER_MATRIX | jq -c 'keys')
-          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
-
   per_cuda:
     name: ${{ matrix.compiler }}
     needs: compute_per_compiler_matrix
     strategy:
       matrix: 
-        compiler: ${{ fromJSON(needs.compute_per_compiler_matrix.COMPILERS) }}
+        compiler: ${{ fromJSON(inputs.compilers) }}
     uses: ./.github/workflows/per_compiler.yml
     with:
-      per_compiler_matrix: ${{ toJSON(fromJSON(needs.compute_per_compiler_matrix.PER_COMPILER_MATRIX)[matrix.compiler]) }}
+      full_matrix: ${{ inputs.full_matrix }}
       compiler_name: ${{ matrix.compiler }}
+      cuda_version: ${{ inputs.cuda_version }}
 
 
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 4ee78bb54d8..aafac2a2db3 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -22,8 +22,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       BUILD_MATRIX:  ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
-      PER_CUDA_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_MATRIX }}
       CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
+      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -33,17 +33,19 @@ jobs:
           cat ./ci/matrix.yaml
           BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)]' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_MATRIX=$(./ci/get_groups.sh $BUILD_MATRIX cuda)
-          echo "PER_CUDA_MATRIX=$PER_CUDA_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          CUDA_VERSIONS=$(echo $PER_CUDA_MATRIX | jq -c 'keys')
+          CUDA_VERSIONS=$(echo $BUILD_MATRIX | jq -c '[.[] | .cuda] | unique')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          COMPILERS=$(echo $BUILD_MATRIX | jq -c '[.[] | .compiler.name] | unique')
+          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
           
   thrust:
+    name: Thrust CUDA ${{ matrix.cuda_version }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
     with:
-      per_cuda_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_MATRIX)[matrix.cuda_version]) }}
-      cuda_version: ${{ matrix.cuda_version }}
\ No newline at end of file
+      cuda_version: ${{ matrix.cuda_version }}
+      full_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      compilers: ${{ needs.compute-matrix.outputs.COMPILERS }}
\ No newline at end of file

From a778156a1901eacd537b6b49d9bd7df236267f09 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:22:21 -0500
Subject: [PATCH 178/395] Remove defunct dependency.

---
 .github/workflows/per_cuda.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 080311d3be1..0a4ed7a0f93 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -10,7 +10,6 @@ on:
 jobs:
   per_cuda:
     name: ${{ matrix.compiler }}
-    needs: compute_per_compiler_matrix
     strategy:
       matrix: 
         compiler: ${{ fromJSON(inputs.compilers) }}

From 38689bc790ebaa5e69790b9cc2e213d39e0ec133 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:24:06 -0500
Subject: [PATCH 179/395] Update names and add stub CUB.

---
 .github/workflows/per_compiler.yml      |  1 +
 .github/workflows/test_new_workflow.yml | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 91860d0355a..6adc52e4b4c 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -9,6 +9,7 @@ on:
 
 jobs:
   per_compiler:
+    name: ${{ inputs.compiler_name }}
     runs-on: ubuntu-latest
     steps:
       - name: test
diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index aafac2a2db3..8886d806f57 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -39,7 +39,19 @@ jobs:
           echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
           
   thrust:
-    name: Thrust CUDA ${{ matrix.cuda_version }}
+    name: CUDA ${{ matrix.cuda_version }}
+    needs: compute-matrix
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+    with:
+      cuda_version: ${{ matrix.cuda_version }}
+      full_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      compilers: ${{ needs.compute-matrix.outputs.COMPILERS }}
+
+  cub:
+    name: CUDA ${{ matrix.cuda_version }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:

From 884d817df7a6ecdba9a548144967e9ffd7a065a5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:25:06 -0500
Subject: [PATCH 180/395] Update thrust/cub names.

---
 .github/workflows/test_new_workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/test_new_workflow.yml
index 8886d806f57..197edfdebe6 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/test_new_workflow.yml
@@ -39,7 +39,7 @@ jobs:
           echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
           
   thrust:
-    name: CUDA ${{ matrix.cuda_version }}
+    name: Thrust CUDA ${{ matrix.cuda_version }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:
@@ -51,7 +51,7 @@ jobs:
       compilers: ${{ needs.compute-matrix.outputs.COMPILERS }}
 
   cub:
-    name: CUDA ${{ matrix.cuda_version }}
+    name: CUB CUDA ${{ matrix.cuda_version }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:

From 2eb14b80e902e879f08192b93494062589f46829 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:34:49 -0500
Subject: [PATCH 181/395] Make per_compiler a matrix.

---
 .github/workflows/per_compiler.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 6adc52e4b4c..be9a90494ac 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -8,12 +8,27 @@ on:
       compiler_name: {type: string, required: true}
 
 jobs:
+
+  compute_per_compiler_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      MATRIX:  ${{ steps.compute_per_compiler_matrix.outputs.MATRIX }}
+    steps:
+      - name: Compute Matrix
+        id: compute_per_compiler_matrix
+        run: |
+          BUILD_MATRIX=$(echo ${{ inputs.full_matrix }} | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
+
   per_compiler:
     name: ${{ inputs.compiler_name }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix: 
+        include: ${{ fromJSON(needs.compute_per_compiler_matrix.outputs.MATRIX) }}
     steps:
       - name: test
         run: |
-          echo "CUDA: ${{inputs.cuda_version}} Compiler: ${{ inputs.compiler_name }}"
+          echo "CUDA: ${{matrix.cuda_version}} Compiler: ${{ matrix.compiler.name }} ${{ matrix.compiler.version }}"
 
 

From c7da9e2724f2cce3bd29f9a5842604cbc9a9f117 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:37:06 -0500
Subject: [PATCH 182/395] Fix matrix variable name.

---
 .github/workflows/per_compiler.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index be9a90494ac..1fa5919665e 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -10,6 +10,7 @@ on:
 jobs:
 
   compute_per_compiler_matrix:
+    name: Compute ${{inputs.compiler_name}} matrix  
     runs-on: ubuntu-latest
     outputs:
       MATRIX:  ${{ steps.compute_per_compiler_matrix.outputs.MATRIX }}
@@ -17,8 +18,8 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          BUILD_MATRIX=$(echo ${{ inputs.full_matrix }} | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          MATRIX=$(echo ${{ inputs.full_matrix }} | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
     name: ${{ inputs.compiler_name }}

From 79cf0e67046fc4c0a5600f12e3c94ef4c6b9a29b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:39:33 -0500
Subject: [PATCH 183/395] Store matrix to envrionment variable first.

---
 .github/workflows/per_compiler.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 1fa5919665e..3d762c7561a 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,7 +18,8 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          MATRIX=$(echo ${{ inputs.full_matrix }} | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+          FULL_MATRIX= "${{ inputs.full_matrix }}"
+          MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
           echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:

From 57b49dd5c05de0b429c4232fe9236303e1876820 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:46:34 -0500
Subject: [PATCH 184/395] Single quotes the full matrix.

---
 .github/workflows/per_compiler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 3d762c7561a..72ca5d6ec89 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX= "${{ inputs.full_matrix }}"
+          FULL_MATRIX= '${{ inputs.full_matrix }}'
           MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
           echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 

From 3129f60cbba299a8ece4f5dae2e3d0c59e332b8c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:52:02 -0500
Subject: [PATCH 185/395] Test is passing matrix is working correctly.

---
 .github/workflows/per_compiler.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 72ca5d6ec89..c7a23c8093b 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -19,8 +19,10 @@ jobs:
         id: compute_per_compiler_matrix
         run: |
           FULL_MATRIX= '${{ inputs.full_matrix }}'
-          MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-          echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
+          echo $FULL_MATRIX
+          
+        #MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+        #echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
     name: ${{ inputs.compiler_name }}

From e0177f28dd1d0c53e73c66c19bd1e648b4389286 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:54:02 -0500
Subject: [PATCH 186/395] Double quotes.

---
 .github/workflows/per_compiler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index c7a23c8093b..db1469c4dc0 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX= '${{ inputs.full_matrix }}'
+          FULL_MATRIX="${{ inputs.full_matrix }}"
           echo $FULL_MATRIX
           
         #MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')

From fa66c83423612ec81c0c31654ea0c7266a0a4f87 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:56:11 -0500
Subject: [PATCH 187/395] Restore computing per compiler matrix.

---
 .github/workflows/per_compiler.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index db1469c4dc0..1beb049bf78 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -13,23 +13,21 @@ jobs:
     name: Compute ${{inputs.compiler_name}} matrix  
     runs-on: ubuntu-latest
     outputs:
-      MATRIX:  ${{ steps.compute_per_compiler_matrix.outputs.MATRIX }}
+      PER_COMPILER_MATRIX:  ${{ steps.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX }}
     steps:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
           FULL_MATRIX="${{ inputs.full_matrix }}"
-          echo $FULL_MATRIX
-          
-        #MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-        #echo "MATRIX=$MATRIX" | tee -a "$GITHUB_OUTPUT"
+          PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
     name: ${{ inputs.compiler_name }}
     runs-on: ubuntu-latest
     strategy:
       matrix: 
-        include: ${{ fromJSON(needs.compute_per_compiler_matrix.outputs.MATRIX) }}
+        include: ${{ fromJSON(needs.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX) }}
     steps:
       - name: test
         run: |

From 79d8b0f37ec86371d333962a231de603f82d9500 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 12:59:46 -0500
Subject: [PATCH 188/395] Use toJSON

---
 .github/workflows/per_compiler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 1beb049bf78..3e9eaf87d5f 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX="${{ inputs.full_matrix }}"
+          FULL_MATRIX='${{ toJSON(inputs.full_matrix) }}'
           PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 

From 0f21fece0098aeb1e9dde64ce0f0e1d3bac2c696 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:09:32 -0500
Subject: [PATCH 189/395] Test matrix assignment.

---
 .github/workflows/per_compiler.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 3e9eaf87d5f..c25f1b2f815 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,9 +18,10 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX='${{ toJSON(inputs.full_matrix) }}'
-          PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          FULL_MATRIX="${{ inputs.full_matrix }}"
+          echo $FULL_MATRIX
+        #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+        #echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
     name: ${{ inputs.compiler_name }}

From 5bb4a7bfc1e495da175c270aaa07c7b51bc7d447 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:11:14 -0500
Subject: [PATCH 190/395] test

---
 .github/workflows/per_compiler.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index c25f1b2f815..c8d39477744 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -19,7 +19,8 @@ jobs:
         id: compute_per_compiler_matrix
         run: |
           FULL_MATRIX="${{ inputs.full_matrix }}"
-          echo $FULL_MATRIX
+          TEST=echo $FULL_MATRIX | jq -c .
+          echo $TEST
         #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
         #echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 

From 1649795938b35552e96ee2f0db374fb29348924a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:12:23 -0500
Subject: [PATCH 191/395] test

---
 .github/workflows/per_compiler.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index c8d39477744..8dc4856dfb6 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -19,8 +19,8 @@ jobs:
         id: compute_per_compiler_matrix
         run: |
           FULL_MATRIX="${{ inputs.full_matrix }}"
-          TEST=echo $FULL_MATRIX | jq -c .
-          echo $TEST
+          TEST=echo "$FULL_MATRIX" | jq -c .
+          echo "$TEST"
         #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
         #echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 

From 5472ae27e44b13609388d5f7acd24283ee6c0878 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:13:34 -0500
Subject: [PATCH 192/395] test2

---
 .github/workflows/per_compiler.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 8dc4856dfb6..7a3fd6d62b7 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -21,6 +21,7 @@ jobs:
           FULL_MATRIX="${{ inputs.full_matrix }}"
           TEST=echo "$FULL_MATRIX" | jq -c .
           echo "$TEST"
+
         #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
         #echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 

From 39b7a24fafd1d0b8db064494c4c016d553b79de3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:19:58 -0500
Subject: [PATCH 193/395] Use toJSON

---
 .github/workflows/per_compiler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 7a3fd6d62b7..134fc8b35ea 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,7 +18,7 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX="${{ inputs.full_matrix }}"
+          FULL_MATRIX="${{ toJSON(inputs.full_matrix) }}"
           TEST=echo "$FULL_MATRIX" | jq -c .
           echo "$TEST"
 

From 1651cabafb6f32fe3d416aac1fa8510cf733c54b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:21:32 -0500
Subject: [PATCH 194/395] Don't quote toJSON output

---
 .github/workflows/per_compiler.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 134fc8b35ea..bf11be73ac6 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -18,8 +18,8 @@ jobs:
       - name: Compute Matrix
         id: compute_per_compiler_matrix
         run: |
-          FULL_MATRIX="${{ toJSON(inputs.full_matrix) }}"
-          TEST=echo "$FULL_MATRIX" | jq -c .
+          FULL_MATRIX=${{ toJSON(inputs.full_matrix) }}
+          TEST=$(echo $FULL_MATRIX | jq -c .)
           echo "$TEST"
 
         #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')

From f2ad7bc924550655590c25f66eb3ff632c07086e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:47:43 -0500
Subject: [PATCH 195/395] Restore setting PER_COMPILER_MATRIX.

---
 .github/workflows/per_compiler.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index bf11be73ac6..9fcdf2f8769 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -19,11 +19,8 @@ jobs:
         id: compute_per_compiler_matrix
         run: |
           FULL_MATRIX=${{ toJSON(inputs.full_matrix) }}
-          TEST=$(echo $FULL_MATRIX | jq -c .)
-          echo "$TEST"
-
-        #PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-        #echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
+          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
     name: ${{ inputs.compiler_name }}

From 8c176ee67ca6a1f0e523531e8b84d4a80de31409 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:49:06 -0500
Subject: [PATCH 196/395] Make per_compiler depend on matrix job.

---
 .github/workflows/per_compiler.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 9fcdf2f8769..7383a818cd7 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -23,6 +23,7 @@ jobs:
           echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   per_compiler:
+    needs: compute_per_compiler_matrix
     name: ${{ inputs.compiler_name }}
     runs-on: ubuntu-latest
     strategy:

From 2f29d6044fb181d04eacff2a5457466987c959fb Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:51:15 -0500
Subject: [PATCH 197/395] Fix per_compiler name.

---
 .github/workflows/per_compiler.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index 7383a818cd7..fa36eb38cf2 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -24,7 +24,7 @@ jobs:
 
   per_compiler:
     needs: compute_per_compiler_matrix
-    name: ${{ inputs.compiler_name }}
+    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     runs-on: ubuntu-latest
     strategy:
       matrix: 

From 986ecc8beec878c0708d93a76d562c0701f5c201 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 13:59:24 -0500
Subject: [PATCH 198/395] Remove per_compiler name.

---
 .github/workflows/per_compiler.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/per_compiler.yml
index fa36eb38cf2..ee2c8a12d1d 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/per_compiler.yml
@@ -24,7 +24,6 @@ jobs:
 
   per_compiler:
     needs: compute_per_compiler_matrix
-    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     runs-on: ubuntu-latest
     strategy:
       matrix: 

From 410b7d1b41ff02a71ec35925f78869abfdece9da Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 14:01:14 -0500
Subject: [PATCH 199/395] Rename to per_library.

---
 .github/workflows/{test_new_workflow.yml => per_library.yml} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{test_new_workflow.yml => per_library.yml} (99%)

diff --git a/.github/workflows/test_new_workflow.yml b/.github/workflows/per_library.yml
similarity index 99%
rename from .github/workflows/test_new_workflow.yml
rename to .github/workflows/per_library.yml
index 197edfdebe6..9968304c889 100644
--- a/.github/workflows/test_new_workflow.yml
+++ b/.github/workflows/per_library.yml
@@ -1,6 +1,6 @@
 
 # This is the main workflow that runs on every PR and push to main
-name: pr_split
+name: Per library
 
 defaults:
   run:

From ed09d7f72f21213ce83c6002f4b07c662cd9bc58 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:06:31 -0500
Subject: [PATCH 200/395] Update top-level matrix to explode cuda/host
 compiler.

---
 .github/workflows/per_cuda.yml    | 16 ++++++++--------
 .github/workflows/per_library.yml | 20 ++++++--------------
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 0a4ed7a0f93..9350e1c78f3 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -4,19 +4,19 @@ on:
   workflow_call:
     inputs:
       cuda_version: {type: string, required: true}
-      full_matrix: {type: string, required: true}
-      compilers: {type: string, required: true}
+      per_cuda_compiler_matrix: {type: string, required: true}
+      compiler: {type: string, required: true}
 
 jobs:
   per_cuda:
     name: ${{ matrix.compiler }}
     strategy:
       matrix: 
-        compiler: ${{ fromJSON(inputs.compilers) }}
-    uses: ./.github/workflows/per_compiler.yml
-    with:
-      full_matrix: ${{ inputs.full_matrix }}
-      compiler_name: ${{ matrix.compiler }}
-      cuda_version: ${{ inputs.cuda_version }}
+        include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: test
+        run: |
+          echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
 
 
diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index 9968304c889..b042cc3d123 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -24,6 +24,7 @@ jobs:
       BUILD_MATRIX:  ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
       CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
       COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -37,6 +38,8 @@ jobs:
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
           COMPILERS=$(echo $BUILD_MATRIX | jq -c '[.[] | .compiler.name] | unique')
           echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_COMPILER_MATRIX=$(echo $BUILD_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
+          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
           
   thrust:
     name: Thrust CUDA ${{ matrix.cuda_version }}
@@ -45,19 +48,8 @@ jobs:
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       cuda_version: ${{ matrix.cuda_version }}
-      full_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      compilers: ${{ needs.compute-matrix.outputs.COMPILERS }}
-
-  cub:
-    name: CUB CUDA ${{ matrix.cuda_version }}
-    needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
-    strategy:
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-    with:
-      cuda_version: ${{ matrix.cuda_version }}
-      full_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      compilers: ${{ needs.compute-matrix.outputs.COMPILERS }}
\ No newline at end of file
+      compiler: ${{ needs.compute-matrix.outputs.COMPILERS }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format({0}-{1}, matrix.cuda_version, matrix.compiler) ]) }}

From 3d8f88551e5310f9a61a228c5c1a683c65166359 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:08:32 -0500
Subject: [PATCH 201/395] Missing quotes in format call

---
 .github/workflows/per_library.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index b042cc3d123..41cecc9a538 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -52,4 +52,4 @@ jobs:
     with:
       cuda_version: ${{ matrix.cuda_version }}
       compiler: ${{ needs.compute-matrix.outputs.COMPILERS }}
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format({0}-{1}, matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}

From 979561517e1626cb2c648aa5ffbd2a59a500be45 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:24:58 -0500
Subject: [PATCH 202/395] Fix per_cuda name.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 9350e1c78f3..404ae39db19 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   per_cuda:
-    name: ${{ matrix.compiler }}
+    name: ${{ inputs.compiler }}
     strategy:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}

From 3edc03cd477f7e07a7325dee4ce1a7040c77278f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:44:05 -0500
Subject: [PATCH 203/395] Fix passed compiler.

---
 .github/workflows/per_library.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index 41cecc9a538..d52d6a5ef3b 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -51,5 +51,6 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       cuda_version: ${{ matrix.cuda_version }}
-      compiler: ${{ needs.compute-matrix.outputs.COMPILERS }}
+      compiler: ${{ matrix.compiler }}
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+

From 47cb54f5db9c635304f2a0effbaed30ac1954415 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:50:21 -0500
Subject: [PATCH 204/395] Fix thrust name.

---
 .github/workflows/per_library.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index d52d6a5ef3b..4a52417395c 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -42,7 +42,7 @@ jobs:
           echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
           
   thrust:
-    name: Thrust CUDA ${{ matrix.cuda_version }}
+    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:

From c7d4168271d936dce3beec3a44521b915711209b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:52:34 -0500
Subject: [PATCH 205/395] Update per_cuda name.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 404ae39db19..d0684a913af 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   per_cuda:
-    name: ${{ inputs.compiler }}
+    name: ${{ matrix.compiler }}${{ matrix.compiler.version }}
     strategy:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}

From 06ba8861c071a5b21508ec7d38118ec2b2faad5a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 16:55:07 -0500
Subject: [PATCH 206/395] Fix per cuda name again.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index d0684a913af..8e40666c0b9 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   per_cuda:
-    name: ${{ matrix.compiler }}${{ matrix.compiler.version }}
+    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}
     strategy:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}

From af81f0f14030d1796f52b477ca626fb55c97412f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 17:00:19 -0500
Subject: [PATCH 207/395] Add C++ dialect and CPU to name.

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 8e40666c0b9..590ec3ba955 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   per_cuda:
-    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}
+    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
     strategy:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}

From fa956365ffe0646b17d2304d9ceb3bb0e0934e68 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 17:05:10 -0500
Subject: [PATCH 208/395] Add CUB/libcu++.

---
 .github/workflows/per_library.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index 4a52417395c..265e6537e3b 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -54,3 +54,29 @@ jobs:
       compiler: ${{ matrix.compiler }}
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
 
+  cub:
+    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      cuda_version: ${{ matrix.cuda_version }}
+      compiler: ${{ matrix.compiler }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+  
+  libcudacxx:
+    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      cuda_version: ${{ matrix.cuda_version }}
+      compiler: ${{ matrix.compiler }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+

From 3a21bfb5dd24d4c6079a73b2bf06f7bbd2848390 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 17:06:26 -0500
Subject: [PATCH 209/395] Fix libcu++ name

---
 .github/workflows/per_library.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index 265e6537e3b..e84b357f5f8 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -68,7 +68,7 @@ jobs:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
   
   libcudacxx:
-    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/per_cuda.yml
     strategy:

From c5a9365ef1e9629dba442f7b5a742d124fbcc6a3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 17:09:39 -0500
Subject: [PATCH 210/395] Add build/test

---
 .github/workflows/per_cuda.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 590ec3ba955..8b795fa584b 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,8 +8,19 @@ on:
       compiler: {type: string, required: true}
 
 jobs:
-  per_cuda:
-    name: ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
+  build:
+    name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
+    strategy:
+      matrix: 
+        include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: test
+        run: |
+          echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
+  test:
+    needs: build
+    name: Test ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
     strategy:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}

From ceee7805c18e874a67781b937434c7e5823aeaee Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:13:24 -0500
Subject: [PATCH 211/395] Make per_cuda call another build-and-test workflow.

---
 .github/workflows/build-and-test.yml | 28 +++++++++++++++
 .github/workflows/per_cuda.yml       | 51 ++++++++++++++++++----------
 2 files changed, 61 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/build-and-test.yml

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
new file mode 100644
index 00000000000..048c0e8b3a5
--- /dev/null
+++ b/.github/workflows/build-and-test.yml
@@ -0,0 +1,28 @@
+name: Base CCCL workflow
+
+on:
+  workflow_call:
+    inputs:
+      cuda_version: {type: string, required: true}
+      compiler: {type: string, required: true}
+      compiler_version: {type: string, required: true}
+      std: {type: string, required: true}
+      gpu_build_archs: {type: string, required: true}
+      cpu: {type: string, required: true}
+      build_script: {type: string, required: false}
+      test_script: {type: string, required: false }
+
+jobs:
+  build:
+    if: github.repository == 'NVIDIA/cccl'
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      run: "echo test"
+  test:
+    needs: build
+    if: github.repository == 'NVIDIA/cccl'
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      run: "echo test"
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 8b795fa584b..f7c37987c9e 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,26 +8,41 @@ on:
       compiler: {type: string, required: true}
 
 jobs:
-  build:
-    name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
+  build_and_test:
     strategy:
+      fail-fast: false
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: test
-        run: |
-          echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
-  test:
-    needs: build
-    name: Test ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
-    strategy:
-      matrix: 
-        include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: test
-        run: |
-          echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
+    uses: ./.github/workflows/build-and-test.yml
+    with:
+      cuda_version: ${{ matrix.cuda_version }}
+      compiler: ${{ matrix.compiler }}
+      compiler_version: ${{ matrix.compiler_version }}
+      std: ${{ matrix.std }}
+      cpu: ${{ matrix.cpu }}
+      gpu_build_archs: ${{ matrix.gpu_build_archs }}
+
+
+  #build:
+  #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
+  #  strategy:
+  #    matrix: 
+  #      include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
+  #  runs-on: ubuntu-latest
+  #  steps:
+  #    - name: test
+  #      run: |
+  #        echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
+  #test:
+  #  needs: build
+  #  name: Test ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
+  #  strategy:
+  #    matrix: 
+  #      include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
+  #  runs-on: ubuntu-latest
+  #  steps:
+  #    - name: test
+  #      run: |
+  #        echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
 
 

From 537de60f44654ee9d06af449a45c38d001b2b21a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:15:28 -0500
Subject: [PATCH 212/395] Fix steps.

---
 .github/workflows/build-and-test.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 048c0e8b3a5..582d223c679 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -18,11 +18,15 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      run: "echo test"
+      - name: test
+        run: | 
+          "echo test"
   test:
     needs: build
     if: github.repository == 'NVIDIA/cccl'
     name: Test
     runs-on: ubuntu-latest
     steps:
-      run: "echo test"
+      - name: test
+        run: | 
+          "echo test"

From e5137bd0e1cb89728fcebbc575deffee8005b59e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:24:07 -0500
Subject: [PATCH 213/395] Remove unneccessary passing of cuda/compiler to
 per_cuda.

---
 .github/workflows/build-and-test.yml | 8 +++-----
 .github/workflows/per_cuda.yml       | 2 --
 .github/workflows/per_library.yml    | 6 ------
 3 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 582d223c679..bbe812c07bc 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1,4 +1,4 @@
-name: Base CCCL workflow
+name: build and test
 
 on:
   workflow_call:
@@ -14,8 +14,7 @@ on:
 
 jobs:
   build:
-    if: github.repository == 'NVIDIA/cccl'
-    name: Build
+    name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
       - name: test
@@ -23,8 +22,7 @@ jobs:
           "echo test"
   test:
     needs: build
-    if: github.repository == 'NVIDIA/cccl'
-    name: Test
+    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}
     runs-on: ubuntu-latest
     steps:
       - name: test
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index f7c37987c9e..b780555fbff 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -3,9 +3,7 @@ name: Per CUDA version
 on:
   workflow_call:
     inputs:
-      cuda_version: {type: string, required: true}
       per_cuda_compiler_matrix: {type: string, required: true}
-      compiler: {type: string, required: true}
 
 jobs:
   build_and_test:
diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
index e84b357f5f8..05f654955d4 100644
--- a/.github/workflows/per_library.yml
+++ b/.github/workflows/per_library.yml
@@ -50,8 +50,6 @@ jobs:
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      cuda_version: ${{ matrix.cuda_version }}
-      compiler: ${{ matrix.compiler }}
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
 
   cub:
@@ -63,8 +61,6 @@ jobs:
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      cuda_version: ${{ matrix.cuda_version }}
-      compiler: ${{ matrix.compiler }}
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
   
   libcudacxx:
@@ -76,7 +72,5 @@ jobs:
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      cuda_version: ${{ matrix.cuda_version }}
-      compiler: ${{ matrix.compiler }}
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
 

From 5ec9e86a44651de00182f0365cf20ed6aa5f88d2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:26:13 -0500
Subject: [PATCH 214/395] missing brace

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index bbe812c07bc..7e2908adf86 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,7 +22,7 @@ jobs:
           "echo test"
   test:
     needs: build
-    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}
+    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
       - name: test

From 133b46f5ccdc3b5551977a66ac543f49d9522940 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:26:37 -0500
Subject: [PATCH 215/395] Remove space.

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7e2908adf86..03beb0368f3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   build:
-    name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}}
+    name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
       - name: test
@@ -22,7 +22,7 @@ jobs:
           "echo test"
   test:
     needs: build
-    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++ ${{inputs.std}}/${{inputs.cpu}}
+    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
       - name: test

From db1cc0b6cde8fc022dce43a1415462527ae34d88 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:30:22 -0500
Subject: [PATCH 216/395] test hang.

---
 .github/workflows/per_cuda.yml | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index b780555fbff..cab63951c7c 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -7,18 +7,24 @@ on:
 
 jobs:
   build_and_test:
-    strategy:
-      fail-fast: false
-      matrix: 
-        include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-    uses: ./.github/workflows/build-and-test.yml
-    with:
-      cuda_version: ${{ matrix.cuda_version }}
-      compiler: ${{ matrix.compiler }}
-      compiler_version: ${{ matrix.compiler_version }}
-      std: ${{ matrix.std }}
-      cpu: ${{ matrix.cpu }}
-      gpu_build_archs: ${{ matrix.gpu_build_archs }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: test
+        run: |
+          echo "test"
+
+    #strategy:
+    #  fail-fast: false
+    #  matrix: 
+    #    include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
+    #uses: ./.github/workflows/build-and-test.yml
+    #with:
+    #  cuda_version: ${{ matrix.cuda_version }}
+    #  compiler: ${{ matrix.compiler }}
+    #  compiler_version: ${{ matrix.compiler_version }}
+    #  std: ${{ matrix.std }}
+    #  cpu: ${{ matrix.cpu }}
+    #  gpu_build_archs: ${{ matrix.gpu_build_archs }}
 
 
   #build:

From 35508bb4c32a080d3e743285a2eb2f2c353a2a47 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:37:32 -0500
Subject: [PATCH 217/395] Test hanging workflow

---
 .github/workflows/per_cuda.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index cab63951c7c..4e638443c2c 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -7,11 +7,14 @@ on:
 
 jobs:
   build_and_test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: test
-        run: |
-          echo "test"
+    uses: ./.github/workflows/build-and-test.yml
+    with:
+      cuda_version: "11.0"
+      compiler: "gcc"
+      compiler_version: "7"
+      std: 11
+      cpu: amd64
+      gpu_build_archs: "50,60,70,80"
 
     #strategy:
     #  fail-fast: false

From 8a451d619ad0f21690bba8d5b393f336909289bc Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:39:37 -0500
Subject: [PATCH 218/395] Fix echo

---
 .github/workflows/build-and-test.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 03beb0368f3..2de7b7947f3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1,5 +1,9 @@
 name: build and test
 
+defaults:
+  run:
+    shell: bash
+
 on:
   workflow_call:
     inputs:
@@ -19,7 +23,7 @@ jobs:
     steps:
       - name: test
         run: | 
-          "echo test"
+          echo "test"
   test:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
@@ -27,4 +31,4 @@ jobs:
     steps:
       - name: test
         run: | 
-          "echo test"
+          echo "test"

From 4dff3927233f078be8270b02914ba9f5e9976e41 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:41:47 -0500
Subject: [PATCH 219/395] Fix passed matrix entries

---
 .github/workflows/per_cuda.yml | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 4e638443c2c..b341105a642 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -8,26 +8,18 @@ on:
 jobs:
   build_and_test:
     uses: ./.github/workflows/build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
     with:
-      cuda_version: "11.0"
-      compiler: "gcc"
-      compiler_version: "7"
-      std: 11
-      cpu: amd64
-      gpu_build_archs: "50,60,70,80"
+      cuda_version: ${{ matrix.cuda }}
+      compiler: ${{ matrix.compiler.name }}
+      compiler_version: ${{ matrix.compiler.version }}
+      std: ${{ matrix.std }}
+      cpu: ${{ matrix.cpu }}
+      gpu_build_archs: ${{ matrix.gpu_build_archs }}
 
-    #strategy:
-    #  fail-fast: false
-    #  matrix: 
-    #    include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-    #uses: ./.github/workflows/build-and-test.yml
-    #with:
-    #  cuda_version: ${{ matrix.cuda_version }}
-    #  compiler: ${{ matrix.compiler }}
-    #  compiler_version: ${{ matrix.compiler_version }}
-    #  std: ${{ matrix.std }}
-    #  cpu: ${{ matrix.cpu }}
-    #  gpu_build_archs: ${{ matrix.gpu_build_archs }}
 
 
   #build:

From 2c9c159d03b38e27a868e2f523bb3232f0efc1c1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:45:45 -0500
Subject: [PATCH 220/395] Test why is it hanging

---
 .github/workflows/per_cuda.yml | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index b341105a642..2d255f804a8 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -9,18 +9,15 @@ jobs:
   build_and_test:
     uses: ./.github/workflows/build-and-test.yml
     strategy:
-      fail-fast: false
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
     with:
-      cuda_version: ${{ matrix.cuda }}
-      compiler: ${{ matrix.compiler.name }}
-      compiler_version: ${{ matrix.compiler.version }}
-      std: ${{ matrix.std }}
-      cpu: ${{ matrix.cpu }}
-      gpu_build_archs: ${{ matrix.gpu_build_archs }}
-
-
+      cuda_version: "11.0"
+      compiler: "gcc"
+      compiler_version: "7"
+      std: "11"
+      cpu: "amd64"
+      gpu_build_archs: "70;75;80"
 
   #build:
   #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}

From a36c8fd50f46ac018c9237a3fb61d563503bf73c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:47:37 -0500
Subject: [PATCH 221/395] Test cuda_version

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 2d255f804a8..73434a3f56a 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -12,7 +12,7 @@ jobs:
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
     with:
-      cuda_version: "11.0"
+      cuda_version: ${{ matrix.cuda }}
       compiler: "gcc"
       compiler_version: "7"
       std: "11"

From f094bbf7e88ae84afef9281ade14e847a993f869 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:51:23 -0500
Subject: [PATCH 222/395] test compiler.name

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 73434a3f56a..a7778de81e5 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -13,7 +13,7 @@ jobs:
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
     with:
       cuda_version: ${{ matrix.cuda }}
-      compiler: "gcc"
+      compiler: ${{ matrix.compiler.name }}
       compiler_version: "7"
       std: "11"
       cpu: "amd64"

From ecd85966ae22317f233a8a9b941caae85fcf1804 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 19:53:30 -0500
Subject: [PATCH 223/395] Test passing all matrix fields.

---
 .github/workflows/per_cuda.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index a7778de81e5..0e13a3dfaae 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -14,11 +14,10 @@ jobs:
     with:
       cuda_version: ${{ matrix.cuda }}
       compiler: ${{ matrix.compiler.name }}
-      compiler_version: "7"
-      std: "11"
-      cpu: "amd64"
-      gpu_build_archs: "70;75;80"
-
+      compiler_version: ${{ matrix.compiler.version }}
+      std: ${{ matrix.std }}
+      gpu_build_archs: ${{ matrix.gpu_build_archs }}
+      cpu: ${{ matrix.cpu }}
   #build:
   #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
   #  strategy:

From 76d4da8f67296e0f84683872fb141c903d81e173 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:02:51 -0500
Subject: [PATCH 224/395] remove build archs

---
 .github/workflows/per_cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 0e13a3dfaae..265ef0494d9 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -16,7 +16,7 @@ jobs:
       compiler: ${{ matrix.compiler.name }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
-      gpu_build_archs: ${{ matrix.gpu_build_archs }}
+      gpu_build_archs: "70"
       cpu: ${{ matrix.cpu }}
   #build:
   #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}

From 3a505211bdacb624b69171d91f16cd91f623a8a4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:17:01 -0500
Subject: [PATCH 225/395] Add random sleep to build step.

---
 .github/workflows/build-and-test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2de7b7947f3..50586c6a05f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,9 @@ jobs:
     steps:
       - name: test
         run: | 
-          echo "test"
+          SLEEP_TIME=$RANDOM % 15
+          echo "Sleeping for $SLEEP_TIME seconds"
+          sleep $SLEEP_TIME
   test:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}

From 7502a288e6ed95768c6e7d77ecf61088696e7fdc Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:19:17 -0500
Subject: [PATCH 226/395] Missing paren

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 50586c6a05f..1ebe48f055c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: test
         run: | 
-          SLEEP_TIME=$RANDOM % 15
+          SLEEP_TIME=$((RANDOM % 15))
           echo "Sleeping for $SLEEP_TIME seconds"
           sleep $SLEEP_TIME
   test:

From 08d4af16d5796bc3aa975ffca095e4352d37b9c5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:29:54 -0500
Subject: [PATCH 227/395] Make cuda11 sleep for 10m

---
 .github/workflows/build-and-test.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1ebe48f055c..932b4ea4a3b 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,8 +23,9 @@ jobs:
     steps:
       - name: test
         run: | 
-          SLEEP_TIME=$((RANDOM % 15))
-          echo "Sleeping for $SLEEP_TIME seconds"
+          if [[ "${{ inputs.cuda }}" == "11.0" ]]; then
+            SLEEP_TIME=600
+          fi
           sleep $SLEEP_TIME
   test:
     needs: build

From 7ebaa9cf5cd306a4675dc5a9a1f49be8089b40ee Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:34:25 -0500
Subject: [PATCH 228/395] Fix sleep time.

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 932b4ea4a3b..965109f401d 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -24,9 +24,9 @@ jobs:
       - name: test
         run: | 
           if [[ "${{ inputs.cuda }}" == "11.0" ]]; then
-            SLEEP_TIME=600
+            SLEEP_TIME=1000 
+            sleep $SLEEP_TIME
           fi
-          sleep $SLEEP_TIME
   test:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}

From c6787d9985950dad729aa2a889ed8e440cc08a9d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:39:19 -0500
Subject: [PATCH 229/395] Reference the right input...

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 965109f401d..1949833514d 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: test
         run: | 
-          if [[ "${{ inputs.cuda }}" == "11.0" ]]; then
+          if [[ "${{ inputs.cuda_version }}" == "11.0" ]]; then
             SLEEP_TIME=1000 
             sleep $SLEEP_TIME
           fi

From 995ec605a82902196295677df12bbb435b69d9a7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 20:49:40 -0500
Subject: [PATCH 230/395] Make C++14 builds sleep

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1949833514d..f55e3b3db0d 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - name: test
         run: | 
-          if [[ "${{ inputs.cuda_version }}" == "11.0" ]]; then
+          if [[ "${{ inputs.std }}" == "14" ]]; then
             SLEEP_TIME=1000 
             sleep $SLEEP_TIME
           fi

From 671f54c393363538505fbff1d8c0bcd63062440b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 21:00:24 -0500
Subject: [PATCH 231/395] Add a comment that explains the overlap.

---
 .github/workflows/per_cuda.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 265ef0494d9..1a34190e1ea 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -6,6 +6,9 @@ on:
       per_cuda_compiler_matrix: {type: string, required: true}
 
 jobs:
+  # Using a matrix to dispatch to the build-and-test reusable workflow for each build configuration
+  # ensures that the build/test steps can overlap across different configurations. For example,
+  # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
     uses: ./.github/workflows/build-and-test.yml
     strategy:
@@ -16,7 +19,7 @@ jobs:
       compiler: ${{ matrix.compiler.name }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
-      gpu_build_archs: "70"
+      gpu_build_archs: "70" #TODO: Fix this
       cpu: ${{ matrix.cpu }}
   #build:
   #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}

From 25bc3a872527eb264ea86f92764a79422f4d4bff Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 21:00:42 -0500
Subject: [PATCH 232/395] Remove sleep

---
 .github/workflows/build-and-test.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f55e3b3db0d..2de7b7947f3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,10 +23,7 @@ jobs:
     steps:
       - name: test
         run: | 
-          if [[ "${{ inputs.std }}" == "14" ]]; then
-            SLEEP_TIME=1000 
-            sleep $SLEEP_TIME
-          fi
+          echo "test"
   test:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}

From e3558e135bc8a4ef208ab59f0175b4137ea1b20d Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 21:50:27 -0500
Subject: [PATCH 233/395] Restructure workflow files

---
 .../workflows/{ => old}/build_and_test.yml    |   0
 .github/workflows/{ => old}/per_compiler.yml  |   1 -
 .github/workflows/old/pr.yml                  | 124 +++++++++++++++++
 .github/workflows/per_cuda.yml                |  25 +---
 .github/workflows/per_library.yml             |  76 -----------
 .github/workflows/pr.yml                      | 126 ++++++------------
 6 files changed, 165 insertions(+), 187 deletions(-)
 rename .github/workflows/{ => old}/build_and_test.yml (100%)
 rename .github/workflows/{ => old}/per_compiler.yml (99%)
 create mode 100644 .github/workflows/old/pr.yml
 delete mode 100644 .github/workflows/per_library.yml

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/old/build_and_test.yml
similarity index 100%
rename from .github/workflows/build_and_test.yml
rename to .github/workflows/old/build_and_test.yml
diff --git a/.github/workflows/per_compiler.yml b/.github/workflows/old/per_compiler.yml
similarity index 99%
rename from .github/workflows/per_compiler.yml
rename to .github/workflows/old/per_compiler.yml
index ee2c8a12d1d..a1335a808dc 100644
--- a/.github/workflows/per_compiler.yml
+++ b/.github/workflows/old/per_compiler.yml
@@ -8,7 +8,6 @@ on:
       compiler_name: {type: string, required: true}
 
 jobs:
-
   compute_per_compiler_matrix:
     name: Compute ${{inputs.compiler_name}} matrix  
     runs-on: ubuntu-latest
diff --git a/.github/workflows/old/pr.yml b/.github/workflows/old/pr.yml
new file mode 100644
index 00000000000..17ba858ee09
--- /dev/null
+++ b/.github/workflows/old/pr.yml
@@ -0,0 +1,124 @@
+# This is the main workflow that runs on every PR and push to main
+name: pr
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  push:
+    branches:
+      - main
+#      - "pull-request/[0-9]+"
+
+# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
+concurrency:
+  group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  compute-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      BUILD_MATRIX: ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
+      TEST_MATRIX:  ${{ steps.compute-matrix.outputs.TEST_MATRIX }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          cat ./ci/matrix.yaml
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
+          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
+          TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
+          echo "TEST_MATRIX=$TEST_MATRIX" | tee -a "$GITHUB_OUTPUT"
+
+  thrust:
+    needs: compute-matrix
+    uses: ./.github/workflows/build_and_test.yml
+    with:
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
+      build_script: "./ci/build_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
+
+  libcudacxx:
+    needs: compute-matrix
+    uses: ./.github/workflows/build_and_test.yml
+    with:
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
+      build_script: "./ci/build_libcudacxx.sh"
+      test_script: "./ci/test_libcudacxx.sh"
+
+  cub:
+    needs: compute-matrix
+    uses: ./.github/workflows/build_and_test.yml
+    with:
+      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
+      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
+      build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
+
+    #steps:
+#   #   - name: Verify compiler version
+#   #     run: |
+#   #       version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
+#   #       if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
+#   #         echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
+#   #         exit 1
+#   #       fi
+#   #       echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
+    #  - name: Get AWS credentials for sccache bucket
+    #    uses: aws-actions/configure-aws-credentials@v2
+    #    with:
+    #      role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
+    #      aws-region: us-east-2 
+    #      role-duration-seconds: 43200 # 12 hours
+    #  - uses: actions/checkout@v3
+    #    with:
+    #      fetch-depth: 0
+    #      persist-credentials: false
+    #  - name: Build Thrust tests
+    #    env:
+    #      SCCACHE_BUCKET: rapids-sccache-east
+    #      SCCACHE_REGION: us-east-2
+    #      SCCACHE_IDLE_TIMEOUT: 32768
+    #      SCCACHE_S3_USE_SSL: true
+    #      SCCACHE_S3_NO_CREDENTIALS: false
+    #    run: |
+    #      time bash ./ci/build_thrust.sh 
+    #      sccache -s
+    #  - name: tar Thrust build artifacts
+    #    run: |
+    #      tar czf thrust-build-artifacts.tar.gz ./thrust
+    #      echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
+    #  - name: Upload Thrust build artifacts
+    #    uses: actions/upload-artifact@v3
+    #    with:
+    #      name: thrust-build-artifacts
+    #      path: thrust-build-artifacts.tar.gz
+  #test:
+  #  name: Run tests
+  #  needs: build
+  #  runs-on: linux-amd64-gpu-v100-latest-1
+  #  container:
+  #    options: -u root
+  #    image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
+  #    env:
+  #      NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+  #  steps:
+  #    - name: Download Thrust build artifacts
+  #      uses: actions/download-artifact@v3
+  #      with:
+  #        name: thrust-build-artifacts
+  #    - name: Untar Thrust build artifacts
+  #      run: tar xzf thrust-build-artifacts.tar.gz
+  #    - name: Run Thrust tests
+  #      run: | 
+  #        pwd 
+  #        ls
+  #        chmod -R +x thrust/build/bin/*
+  #        ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/per_cuda.yml
index 1a34190e1ea..f8ef91173f0 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/per_cuda.yml
@@ -1,4 +1,4 @@
-name: Per CUDA version
+name: Dispatch build and test
 
 on:
   workflow_call:
@@ -10,6 +10,7 @@ jobs:
   # ensures that the build/test steps can overlap across different configurations. For example,
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
+    name: Build and test ${{matrix.compiler.name}}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     uses: ./.github/workflows/build-and-test.yml
     strategy:
       matrix: 
@@ -21,26 +22,4 @@ jobs:
       std: ${{ matrix.std }}
       gpu_build_archs: "70" #TODO: Fix this
       cpu: ${{ matrix.cpu }}
-  #build:
-  #  name: Build ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
-  #  strategy:
-  #    matrix: 
-  #      include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - name: test
-  #      run: |
-  #        echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
-  #test:
-  #  needs: build
-  #  name: Test ${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++ ${{matrix.std}}/${{matrix.cpu}}
-  #  strategy:
-  #    matrix: 
-  #      include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - name: test
-  #      run: |
-  #        echo "CUDA: ${{ matrix.cuda_version }} Compiler: ${{ matrix.compiler }} ${{ matrix.compiler_version }}"
-
 
diff --git a/.github/workflows/per_library.yml b/.github/workflows/per_library.yml
deleted file mode 100644
index 05f654955d4..00000000000
--- a/.github/workflows/per_library.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-
-# This is the main workflow that runs on every PR and push to main
-name: Per library
-
-defaults:
-  run:
-    shell: bash
-
-on:
-  push:
-    branches:
-      - main
-      - "pull-request/[0-9]+"
-
-# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
-concurrency:
-  group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
-  cancel-in-progress: true
-
-jobs:
-  compute-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      BUILD_MATRIX:  ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
-      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
-      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
-      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Compute matrix
-        id: compute-matrix
-        run: |
-          cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)]' )
-          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          CUDA_VERSIONS=$(echo $BUILD_MATRIX | jq -c '[.[] | .cuda] | unique')
-          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
-          COMPILERS=$(echo $BUILD_MATRIX | jq -c '[.[] | .compiler.name] | unique')
-          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_COMPILER_MATRIX=$(echo $BUILD_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
-          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          
-  thrust:
-    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
-    strategy:
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-
-  cub:
-    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
-    strategy:
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-  
-  libcudacxx:
-    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
-    strategy:
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 17ba858ee09..dc4c5cdae4d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -1,3 +1,4 @@
+
 # This is the main workflow that runs on every PR and push to main
 name: pr
 
@@ -9,7 +10,7 @@ on:
   push:
     branches:
       - main
-#      - "pull-request/[0-9]+"
+      - "pull-request/[0-9]+"
 
 # Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
 concurrency:
@@ -20,8 +21,10 @@ jobs:
   compute-matrix:
     runs-on: ubuntu-latest
     outputs:
-      BUILD_MATRIX: ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
-      TEST_MATRIX:  ${{ steps.compute-matrix.outputs.TEST_MATRIX }}
+      FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
+      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
+      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -29,96 +32,45 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
-          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
-          TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
-          echo "TEST_MATRIX=$TEST_MATRIX" | tee -a "$GITHUB_OUTPUT"
-
+          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)]' )
+          echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
+          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
+          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
+          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          
   thrust:
+    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
 
-  libcudacxx:
+  cub:
+    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_libcudacxx.sh"
-      test_script: "./ci/test_libcudacxx.sh"
-
-  cub:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+  
+  libcudacxx:
+    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
+    uses: ./.github/workflows/per_cuda.yml
+    strategy:
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
 
-    #steps:
-#   #   - name: Verify compiler version
-#   #     run: |
-#   #       version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-#   #       if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
-#   #         echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
-#   #         exit 1
-#   #       fi
-#   #       echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
-    #  - name: Get AWS credentials for sccache bucket
-    #    uses: aws-actions/configure-aws-credentials@v2
-    #    with:
-    #      role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-    #      aws-region: us-east-2 
-    #      role-duration-seconds: 43200 # 12 hours
-    #  - uses: actions/checkout@v3
-    #    with:
-    #      fetch-depth: 0
-    #      persist-credentials: false
-    #  - name: Build Thrust tests
-    #    env:
-    #      SCCACHE_BUCKET: rapids-sccache-east
-    #      SCCACHE_REGION: us-east-2
-    #      SCCACHE_IDLE_TIMEOUT: 32768
-    #      SCCACHE_S3_USE_SSL: true
-    #      SCCACHE_S3_NO_CREDENTIALS: false
-    #    run: |
-    #      time bash ./ci/build_thrust.sh 
-    #      sccache -s
-    #  - name: tar Thrust build artifacts
-    #    run: |
-    #      tar czf thrust-build-artifacts.tar.gz ./thrust
-    #      echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
-    #  - name: Upload Thrust build artifacts
-    #    uses: actions/upload-artifact@v3
-    #    with:
-    #      name: thrust-build-artifacts
-    #      path: thrust-build-artifacts.tar.gz
-  #test:
-  #  name: Run tests
-  #  needs: build
-  #  runs-on: linux-amd64-gpu-v100-latest-1
-  #  container:
-  #    options: -u root
-  #    image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
-  #    env:
-  #      NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-  #  steps:
-  #    - name: Download Thrust build artifacts
-  #      uses: actions/download-artifact@v3
-  #      with:
-  #        name: thrust-build-artifacts
-  #    - name: Untar Thrust build artifacts
-  #      run: tar xzf thrust-build-artifacts.tar.gz
-  #    - name: Run Thrust tests
-  #      run: | 
-  #        pwd 
-  #        ls
-  #        chmod -R +x thrust/build/bin/*
-  #        ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From 41bf24bccc87e5ad5a15882124cf1ed5c463be6f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 21:55:29 -0500
Subject: [PATCH 234/395] Rename per_cuda

---
 .github/workflows/build-and-test.yml                   | 10 ++++++++++
 .../{per_cuda.yml => dispatch-build-and-test.yml}      |  4 ++++
 .github/workflows/pr.yml                               |  6 +++---
 3 files changed, 17 insertions(+), 3 deletions(-)
 rename .github/workflows/{per_cuda.yml => dispatch-build-and-test.yml} (83%)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2de7b7947f3..02632cfbe40 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -24,6 +24,11 @@ jobs:
       - name: test
         run: | 
           echo "test"
+          if [[ -n "${{ inputs.build_script }}" ]]; then
+            ./${{ inputs.build_script }}
+          else
+            echo "No build script provided, skipping."
+          fi
   test:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
@@ -32,3 +37,8 @@ jobs:
       - name: test
         run: | 
           echo "test"
+          if [[ -n "${{ inputs.test_script }}" ]]; then
+            ./${{ inputs.test_script }}
+          else
+            echo "No test script provided, skipping."
+          fi
diff --git a/.github/workflows/per_cuda.yml b/.github/workflows/dispatch-build-and-test.yml
similarity index 83%
rename from .github/workflows/per_cuda.yml
rename to .github/workflows/dispatch-build-and-test.yml
index f8ef91173f0..980921b02f4 100644
--- a/.github/workflows/per_cuda.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -4,6 +4,8 @@ on:
   workflow_call:
     inputs:
       per_cuda_compiler_matrix: {type: string, required: true}
+      build_script: {type: string, required: false}
+      test_script: {type: string, required: false}
 
 jobs:
   # Using a matrix to dispatch to the build-and-test reusable workflow for each build configuration
@@ -22,4 +24,6 @@ jobs:
       std: ${{ matrix.std }}
       gpu_build_archs: "70" #TODO: Fix this
       cpu: ${{ matrix.cpu }}
+      build_script: ${{ inputs.build_script }}
+      test_script: ${{ inputs.test_script }}
 
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index dc4c5cdae4d..83c98c51faf 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -44,7 +44,7 @@ jobs:
   thrust:
     name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
+    uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
@@ -55,7 +55,7 @@ jobs:
   cub:
     name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
+    uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
@@ -66,7 +66,7 @@ jobs:
   libcudacxx:
     name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
     needs: compute-matrix
-    uses: ./.github/workflows/per_cuda.yml
+    uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 

From a5ab85ca7d476934cb9fffcf33ddb090f25ebe29 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 21:57:35 -0500
Subject: [PATCH 235/395] Update step names

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 02632cfbe40..a833427e334 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -21,7 +21,7 @@ jobs:
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
-      - name: test
+      - name: Run build script
         run: | 
           echo "test"
           if [[ -n "${{ inputs.build_script }}" ]]; then
@@ -34,7 +34,7 @@ jobs:
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
     steps:
-      - name: test
+      - name: Run test script
         run: | 
           echo "test"
           if [[ -n "${{ inputs.test_script }}" ]]; then

From 2e16520c8156804191e017d330ad893f8199eeb3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:04:34 -0500
Subject: [PATCH 236/395] Only run test if build succeeded.

---
 .github/workflows/build-and-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a833427e334..f2d6ab0b632 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -33,6 +33,7 @@ jobs:
     needs: build
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
+    if: ${{ needs.build.result == 'success' }}
     steps:
       - name: Run test script
         run: | 

From 5f926573d9d9b200635202bcb5b6901f16ee70ce Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:06:57 -0500
Subject: [PATCH 237/395] Add references to stub build/test scripts.

---
 .github/workflows/pr.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 83c98c51faf..983f61dd5de 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -51,6 +51,8 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
 
   cub:
     name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
@@ -62,6 +64,8 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
   
   libcudacxx:
     name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
@@ -73,4 +77,6 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_libcudacxx.sh"
+      test_script: "./ci/test_libcudacxx.sh"
 

From 910413422b05cc79fe86e60294067f6b9d4b73dd Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:08:39 -0500
Subject: [PATCH 238/395] fail-fast false

---
 .github/workflows/dispatch-build-and-test.yml | 1 +
 .github/workflows/pr.yml                      | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 980921b02f4..bce9669cc1e 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -15,6 +15,7 @@ jobs:
     name: Build and test ${{matrix.compiler.name}}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     uses: ./.github/workflows/build-and-test.yml
     strategy:
+      fail-fast: false
       matrix: 
         include: ${{ fromJSON(inputs.per_cuda_compiler_matrix) }}
     with:
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 983f61dd5de..8f3d1a9e00b 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -46,6 +46,7 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
+      fail-fast: false
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
@@ -59,6 +60,7 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
+      fail-fast: false
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
@@ -72,6 +74,7 @@ jobs:
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
+      fail-fast: false
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}

From b6893d395bd6003c2fd310eae930eac182217695 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:14:05 -0500
Subject: [PATCH 239/395] Flesh out build and test jobs.

---
 .github/workflows/build-and-test.yml | 30 +++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f2d6ab0b632..37d10efe672 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -20,7 +20,21 @@ jobs:
   build:
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
+    #runs-on: linux-${{inputs.cpu}}-cpu16
+    #container: # Fetch the container image to use for the job
+    #  options: -u root
+    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write
+      contents: read
     steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Configure credentials and environment variables for sccache
+        uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
           echo "test"
@@ -31,10 +45,24 @@ jobs:
           fi
   test:
     needs: build
+    if: ${{ needs.build.result == 'success' }}
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
     runs-on: ubuntu-latest
-    if: ${{ needs.build.result == 'success' }}
+    #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
+    #container: # Fetch the container image to use for the job
+    #  options: -u root
+    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    permissions:
+      id-token: write
+      contents: read
     steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Configure credentials and environment variables for sccache
+        uses: ./.github/actions/configure_cccl_sccache
       - name: Run test script
         run: | 
           echo "test"

From 7a2619616af5c82382913747037ce405bf6c2be6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:17:30 -0500
Subject: [PATCH 240/395] Update stub test scripts

---
 ci/test_cub.sh        | 2 +-
 ci/test_libcudacxx.sh | 2 +-
 ci/test_thrust.sh     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/test_cub.sh b/ci/test_cub.sh
index 2782951996b..cb2f15b739e 100755
--- a/ci/test_cub.sh
+++ b/ci/test_cub.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "build cub"
+echo "test cub"
 
 # Require a compiler to be specified
 #if [ $# -ne 1 ]; then
diff --git a/ci/test_libcudacxx.sh b/ci/test_libcudacxx.sh
index 106049915f4..c26fe94c552 100755
--- a/ci/test_libcudacxx.sh
+++ b/ci/test_libcudacxx.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "build libcudacxx"
+echo "test libcudacxx"
 
 # Require a compiler to be specified
 #if [ $# -ne 1 ]; then
diff --git a/ci/test_thrust.sh b/ci/test_thrust.sh
index 4f901e064b4..1433ad049df 100755
--- a/ci/test_thrust.sh
+++ b/ci/test_thrust.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "build thrust"
+echo "test thrust"
 
 # Require a compiler to be specified
 #if [ $# -ne 1 ]; then

From 4bebf4daf4d9fb4a8029b42c7947292d59df9cc0 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:28:34 -0500
Subject: [PATCH 241/395] Factor format call into env

---
 .github/workflows/pr.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 8f3d1a9e00b..288415b1f0e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -50,8 +50,10 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    env:
+      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 
@@ -64,8 +66,10 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    env:
+      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
       build_script: "./ci/build_cub.sh"
       test_script: "./ci/test_cub.sh"
   
@@ -78,8 +82,10 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    env:
+      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
       build_script: "./ci/build_libcudacxx.sh"
       test_script: "./ci/test_libcudacxx.sh"
 

From 165f4316861958bf30b9e26fd583445034db86f3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 30 May 2023 22:35:12 -0500
Subject: [PATCH 242/395] Undo factoring format into env.

---
 .github/workflows/pr.yml | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 288415b1f0e..4a932fd9cb1 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -28,6 +28,9 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+      # PER_CUDA_COMPILER_MATRIX is a JSON object where the keys are the CUDA version and compiler name concatenated and 
+      # the values are the build configurations for that CUDA version and compiler. For example, the values associated with "11.0-gcc" 
+      # are the build configurations for CUDA 11.0 with gcc and clang.
       - name: Compute matrix
         id: compute-matrix
         run: |
@@ -50,10 +53,8 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    env:
-      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 
@@ -66,10 +67,8 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    env:
-      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_cub.sh"
       test_script: "./ci/test_cub.sh"
   
@@ -82,10 +81,8 @@ jobs:
       matrix: 
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    env:
-      CUDA_COMPILER_KEY: ${{ format('{0}-{1}', matrix.cuda_version, matrix.compiler) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ env.CUDA_COMPILER_KEY ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_libcudacxx.sh"
       test_script: "./ci/test_libcudacxx.sh"
 

From 8f0cba57bfadd0ae366eec0423ccc84558dc8452 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 06:41:08 -0500
Subject: [PATCH 243/395] Rename matrix "include:" to "pull-request"

In anticipation of adding nightly testing sections.
---
 .github/workflows/old/pr.yml | 2 +-
 ci/matrix.yaml               | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/old/pr.yml b/.github/workflows/old/pr.yml
index 17ba858ee09..94844d412dc 100644
--- a/.github/workflows/old/pr.yml
+++ b/.github/workflows/old/pr.yml
@@ -29,7 +29,7 @@ jobs:
         id: compute-matrix
         run: |
           cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
+          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.pull_request[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
           echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
           GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
           TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 0e74c5791bc..a68c945a010 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -19,7 +19,9 @@ gpus:
 # gpu_build_archs: The GPU architectures to build for
 # std: The C++ standards to build for
 #    This field is unique as it will generate an independent build/test job for each value
-include:
+
+# Configurations that will run for every PR
+pull-request:
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
@@ -29,8 +31,6 @@ include:
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
 - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
 - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-
- #clang
 - {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}

From 1552fb4cf1c7da6b092a2a60ba8991f9413c5d36 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 07:18:07 -0500
Subject: [PATCH 244/395] Factor out root workflow into reusable workflow.

---
 .github/workflows/linux-build-and-test.yml | 96 ++++++++++++++++++++++
 .github/workflows/pr.yml                   | 71 ++--------------
 2 files changed, 101 insertions(+), 66 deletions(-)
 create mode 100644 .github/workflows/linux-build-and-test.yml

diff --git a/.github/workflows/linux-build-and-test.yml b/.github/workflows/linux-build-and-test.yml
new file mode 100644
index 00000000000..cbcff4cdae6
--- /dev/null
+++ b/.github/workflows/linux-build-and-test.yml
@@ -0,0 +1,96 @@
+name: Run linux build and test
+
+
+on:
+  workflow_call:
+    inputs:
+      matrix_file: 
+        type: string
+        required: true
+      build_type:
+        type: string
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  compute-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
+      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
+      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
+    env:
+      BUILD_TYPE: ${{ github.event.inputs.build_type}}
+      MATRIX_FILE: ${{ github.event.inputs.matrix_file }}}
+    steps:
+      - name: Validate build type
+        run: |
+          if [[ "$BUILD_TYPE" != "pull-request" && "$BUILD_TYPE" != "nightly" ]]; then
+            echo "Invalid build type: $BUILD_TYPE. Must be 'nightly' or 'pull-request'"
+            exit 1
+          fi
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      # PER_CUDA_COMPILER_MATRIX is a JSON object where the keys are the CUDA version and compiler name concatenated and 
+      # the values are the build configurations for that CUDA version and compiler. For example, the values associated with "11.0-gcc" 
+      # are the build configurations for CUDA 11.0 with gcc and clang.
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          echo "MATRIX_FILE=$MATRIX_FILE"
+          cat $MATRIX_FILE
+          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '.[$build_type][] | . as $o | {std: .std[]} + del($o.std)]' )
+          echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
+          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
+          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
+          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
+
+  thrust:
+    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
+
+  cub:
+    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
+  
+  libcudacxx:
+    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_libcudacxx.sh"
+      test_script: "./ci/test_libcudacxx.sh"
+          
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 4a932fd9cb1..3df82fd4507 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -18,71 +18,10 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  compute-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
-      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
-      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
-      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      # PER_CUDA_COMPILER_MATRIX is a JSON object where the keys are the CUDA version and compiler name concatenated and 
-      # the values are the build configurations for that CUDA version and compiler. For example, the values associated with "11.0-gcc" 
-      # are the build configurations for CUDA 11.0 with gcc and clang.
-      - name: Compute matrix
-        id: compute-matrix
-        run: |
-          cat ./ci/matrix.yaml
-          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.include[] | . as $o | {std: .std[]} + del($o.std)]' )
-          echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
-          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
-          COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
-          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
-          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          
-  thrust:
-    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+  linux-build-test:
+    name: Linux
+    uses: ./.github/workflows/linux-build-and-test.yml
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
-
-  cub:
-    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
-  
-  libcudacxx:
-    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_libcudacxx.sh"
-      test_script: "./ci/test_libcudacxx.sh"
+      matrix_file: ./ci/matrix.yaml
+      build_type: "pull-request"
 

From fc460c7cebb204640c888c7d4bb15c1ef9584155 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 07:19:58 -0500
Subject: [PATCH 245/395] Fix inputs to linux-build-and-test

---
 .github/workflows/linux-build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/linux-build-and-test.yml b/.github/workflows/linux-build-and-test.yml
index cbcff4cdae6..fe73a58cf47 100644
--- a/.github/workflows/linux-build-and-test.yml
+++ b/.github/workflows/linux-build-and-test.yml
@@ -24,8 +24,8 @@ jobs:
       COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
       PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
     env:
-      BUILD_TYPE: ${{ github.event.inputs.build_type}}
-      MATRIX_FILE: ${{ github.event.inputs.matrix_file }}}
+      BUILD_TYPE: ${{ inputs.build_type}}
+      MATRIX_FILE: ${{ inputs.matrix_file }}}
     steps:
       - name: Validate build type
         run: |

From c1a0e6642846d71528c9167b3d4f53d81ec0ed80 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 07:22:35 -0500
Subject: [PATCH 246/395] Extra brace.

---
 .github/workflows/linux-build-and-test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/linux-build-and-test.yml b/.github/workflows/linux-build-and-test.yml
index fe73a58cf47..44fb130d040 100644
--- a/.github/workflows/linux-build-and-test.yml
+++ b/.github/workflows/linux-build-and-test.yml
@@ -25,7 +25,7 @@ jobs:
       PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
     env:
       BUILD_TYPE: ${{ inputs.build_type}}
-      MATRIX_FILE: ${{ inputs.matrix_file }}}
+      MATRIX_FILE: ${{ inputs.matrix_file }}
     steps:
       - name: Validate build type
         run: |
@@ -41,8 +41,8 @@ jobs:
       - name: Compute matrix
         id: compute-matrix
         run: |
-          echo "MATRIX_FILE=$MATRIX_FILE"
-          cat $MATRIX_FILE
+          echo "Input matrix file:"
+          cat "$MATRIX_FILE"
           FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '.[$build_type][] | . as $o | {std: .std[]} + del($o.std)]' )
           echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
           CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')

From 33fee4c08bb6c9f024a9016916a18f5df840d4a4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 07:39:55 -0500
Subject: [PATCH 247/395] Fix jq syntax

---
 .github/workflows/linux-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/linux-build-and-test.yml b/.github/workflows/linux-build-and-test.yml
index 44fb130d040..e7820bc7347 100644
--- a/.github/workflows/linux-build-and-test.yml
+++ b/.github/workflows/linux-build-and-test.yml
@@ -43,7 +43,7 @@ jobs:
         run: |
           echo "Input matrix file:"
           cat "$MATRIX_FILE"
-          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '.[$build_type][] | . as $o | {std: .std[]} + del($o.std)]' )
+          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '[ .[$build_type][] | . as $o | {std: .std[]} + del($o.std)]')
           echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
           CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"

From b85151c2710710d32ef41f4683782c381563f0ad Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 07:43:37 -0500
Subject: [PATCH 248/395] Don't give root linux job a name.

---
 .github/workflows/pr.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3df82fd4507..a26afc29962 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -19,7 +19,6 @@ concurrency:
 
 jobs:
   linux-build-test:
-    name: Linux
     uses: ./.github/workflows/linux-build-and-test.yml
     with:
       matrix_file: ./ci/matrix.yaml

From 2c2a7c53bbc9402bed330edacc1e0ebc44bc2a50 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:07:05 -0500
Subject: [PATCH 249/395] Go back to library jobs in root workflow.

Factored compute-matrix into reusable workflow.
---
 .github/workflows/compute-matrix.yml | 66 ++++++++++++++++++++++++++++
 .github/workflows/pr.yml             | 53 ++++++++++++++++++++--
 2 files changed, 115 insertions(+), 4 deletions(-)
 create mode 100644 .github/workflows/compute-matrix.yml

diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
new file mode 100644
index 00000000000..b266b90733a
--- /dev/null
+++ b/.github/workflows/compute-matrix.yml
@@ -0,0 +1,66 @@
+
+name: Compute the matrix for a given build type from the specified matrix file
+
+on:
+  workflow_call:
+    inputs:
+      matrix_file: 
+        type: string
+        required: true
+      build_type:
+        description: "The desired build type. Currently, only 'pull-request' and 'nightly' are supported."
+        type: string
+        required: true
+    outputs:
+      FULL_MATRIX: 
+        description: "The full matrix of build configurations for the given build type."
+        value: ${{ jobs.compute-matrix.outputs.FULL_MATRIX }}
+      CUDA_VERSIONS:
+        description: "The unique CUDA versions in the full matrix."
+        value: ${{ jobs.compute-matrix.outputs.CUDA_VERSIONS }}
+      COMPILERS:
+        description: "The unique compilers in the full matrix."
+        value: ${{ jobs.compute-matrix.outputs.COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX:
+        description: "The matrix of build configurations for each CUDA version and compiler."
+        value: ${{ jobs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  BUILD_TYPE: ${{ inputs.build_type}}
+  MATRIX_FILE: ${{ inputs.matrix_file }}
+
+jobs:
+  compute-matrix:
+    name: Compute $BUILD_TYPE matrix
+    runs-on: ubuntu-latest
+    outputs:
+      FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
+      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
+      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
+    steps:
+      - name: Validate build type
+        run: |
+          if [[ "$BUILD_TYPE" != "pull-request" && "$BUILD_TYPE" != "nightly" ]]; then
+            echo "Invalid build type: $BUILD_TYPE. Must be 'nightly' or 'pull-request'"
+            exit 1
+          fi
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          echo "Input matrix file:"
+          cat "$MATRIX_FILE"
+          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '[ .[$build_type][] | . as $o | {std: .std[]} + del($o.std)]')
+          echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
+          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
+          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
+          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
+          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
\ No newline at end of file
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index a26afc29962..6073c778154 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -17,10 +17,55 @@ concurrency:
   group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
 
+env:
+  BUILD_TYPE: "pull-request"
+  MATRIX_FILE: "./ci/matrix.yaml"
+
 jobs:
-  linux-build-test:
-    uses: ./.github/workflows/linux-build-and-test.yml
+  compute-matrix:
+    uses: ./.github/workflows/compute-matrix.yml
+    with:
+      matrix_file: ${{ env.MATRIX_FILE }}
+      build_type: ${{ env.BUILD_TYPE }}
+  
+  thrust:
+    name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      matrix_file: ./ci/matrix.yaml
-      build_type: "pull-request"
+      per_cuda_compiler_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX) }}
+      build_script: "./ci/build_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
 
+  cub:
+    name: CUB-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
+  
+  libcudacxx:
+    name: libcudacxx-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_libcudacxx.sh"
+      test_script: "./ci/test_libcudacxx.sh" 

From 8504a993f0ec92e1e109bbd97e0b5bf816a22cf1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:22:31 -0500
Subject: [PATCH 250/395] Fix env

---
 .github/workflows/pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 6073c778154..0191e5cef8c 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -25,8 +25,8 @@ jobs:
   compute-matrix:
     uses: ./.github/workflows/compute-matrix.yml
     with:
-      matrix_file: ${{ env.MATRIX_FILE }}
-      build_type: ${{ env.BUILD_TYPE }}
+      matrix_file: $BUILD_TYPE
+      build_type: $MATRIX_FILE
   
   thrust:
     name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}

From af1ff44af77b0c2e98afd11d00b685535fd575d9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:32:49 -0500
Subject: [PATCH 251/395] Correct swapped inputs.

---
 .github/workflows/pr.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 0191e5cef8c..bc17af01a8f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -25,9 +25,9 @@ jobs:
   compute-matrix:
     uses: ./.github/workflows/compute-matrix.yml
     with:
-      matrix_file: $BUILD_TYPE
-      build_type: $MATRIX_FILE
-  
+      matrix_file: $MATRIX_FILE
+      build_type: $BUILD_TYPE
+
   thrust:
     name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
     needs: compute-matrix

From 0faba7ba3938bf481f46fa38238ec6b730de7e08 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:42:58 -0500
Subject: [PATCH 252/395] Use more specific env accessor.

---
 .github/workflows/pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index bc17af01a8f..3c2469bbe71 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -25,8 +25,8 @@ jobs:
   compute-matrix:
     uses: ./.github/workflows/compute-matrix.yml
     with:
-      matrix_file: $MATRIX_FILE
-      build_type: $BUILD_TYPE
+      matrix_file: ${{ env.MATRIX_FILE }}
+      build_type: ${{ env.BUILD_TYPE }}
 
   thrust:
     name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}

From 42b4427972921cd411156c6d6919099386e8bc98 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:47:25 -0500
Subject: [PATCH 253/395] Hardcode matrix/build type instead of env.

---
 .github/workflows/pr.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3c2469bbe71..e1b97eb9351 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -17,16 +17,12 @@ concurrency:
   group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
   cancel-in-progress: true
 
-env:
-  BUILD_TYPE: "pull-request"
-  MATRIX_FILE: "./ci/matrix.yaml"
-
 jobs:
   compute-matrix:
     uses: ./.github/workflows/compute-matrix.yml
     with:
-      matrix_file: ${{ env.MATRIX_FILE }}
-      build_type: ${{ env.BUILD_TYPE }}
+      matrix_file: "./ci/matrix.yaml"
+      build_type: "pull-request"
 
   thrust:
     name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}

From 4e6582ca6a0ed3eb842924c13589bcbaa628e765 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:51:04 -0500
Subject: [PATCH 254/395] Fix compute-matrix name.

---
 .github/workflows/compute-matrix.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
index b266b90733a..e1dbf350837 100644
--- a/.github/workflows/compute-matrix.yml
+++ b/.github/workflows/compute-matrix.yml
@@ -35,7 +35,7 @@ env:
 
 jobs:
   compute-matrix:
-    name: Compute $BUILD_TYPE matrix
+    name: Compute ${{env.BUILD_TYPE}} matrix
     runs-on: ubuntu-latest
     outputs:
       FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}

From 138577fcd867b13887e81f98fd6e457ecefe7be9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:53:13 -0500
Subject: [PATCH 255/395] Access inputs instead of env for name.

---
 .github/workflows/compute-matrix.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
index e1dbf350837..638b688d3a9 100644
--- a/.github/workflows/compute-matrix.yml
+++ b/.github/workflows/compute-matrix.yml
@@ -35,7 +35,7 @@ env:
 
 jobs:
   compute-matrix:
-    name: Compute ${{env.BUILD_TYPE}} matrix
+    name: Compute ${{inputs.build_type}} matrix
     runs-on: ubuntu-latest
     outputs:
       FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}

From c103acec7d98c13ccef27f0e45e2e466bca332f4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 08:57:58 -0500
Subject: [PATCH 256/395] Delete unused linux-build-and-test workflow

---
 .github/workflows/linux-build-and-test.yml | 96 ----------------------
 1 file changed, 96 deletions(-)
 delete mode 100644 .github/workflows/linux-build-and-test.yml

diff --git a/.github/workflows/linux-build-and-test.yml b/.github/workflows/linux-build-and-test.yml
deleted file mode 100644
index e7820bc7347..00000000000
--- a/.github/workflows/linux-build-and-test.yml
+++ /dev/null
@@ -1,96 +0,0 @@
-name: Run linux build and test
-
-
-on:
-  workflow_call:
-    inputs:
-      matrix_file: 
-        type: string
-        required: true
-      build_type:
-        type: string
-        required: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  compute-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
-      CUDA_VERSIONS: ${{ steps.compute-matrix.outputs.CUDA_VERSIONS }}
-      COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
-      PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
-    env:
-      BUILD_TYPE: ${{ inputs.build_type}}
-      MATRIX_FILE: ${{ inputs.matrix_file }}
-    steps:
-      - name: Validate build type
-        run: |
-          if [[ "$BUILD_TYPE" != "pull-request" && "$BUILD_TYPE" != "nightly" ]]; then
-            echo "Invalid build type: $BUILD_TYPE. Must be 'nightly' or 'pull-request'"
-            exit 1
-          fi
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      # PER_CUDA_COMPILER_MATRIX is a JSON object where the keys are the CUDA version and compiler name concatenated and 
-      # the values are the build configurations for that CUDA version and compiler. For example, the values associated with "11.0-gcc" 
-      # are the build configurations for CUDA 11.0 with gcc and clang.
-      - name: Compute matrix
-        id: compute-matrix
-        run: |
-          echo "Input matrix file:"
-          cat "$MATRIX_FILE"
-          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '[ .[$build_type][] | . as $o | {std: .std[]} + del($o.std)]')
-          echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
-          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
-          COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
-          echo "COMPILERS=$COMPILERS" | tee -a "$GITHUB_OUTPUT"
-          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
-          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
-
-  thrust:
-    name: Thrust - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
-
-  cub:
-    name: CUB - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
-  
-  libcudacxx:
-    name: libcudacxx - CUDA ${{ matrix.cuda_version }} - ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_libcudacxx.sh"
-      test_script: "./ci/test_libcudacxx.sh"
-          
\ No newline at end of file

From 63b64923557181bd2aea686345ae7cf5cb05a63b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:06:07 -0500
Subject: [PATCH 257/395] Update names

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 .github/workflows/pr.yml                      | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index bce9669cc1e..89f54962a97 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -12,7 +12,7 @@ jobs:
   # ensures that the build/test steps can overlap across different configurations. For example,
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
-    name: Build and test ${{matrix.compiler.name}}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
+    #name: Build and test ${{matrix.compiler.name}}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
     uses: ./.github/workflows/build-and-test.yml
     strategy:
       fail-fast: false
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e1b97eb9351..523f23eff79 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -25,7 +25,7 @@ jobs:
       build_type: "pull-request"
 
   thrust:
-    name: Thrust-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
@@ -39,7 +39,7 @@ jobs:
       test_script: "./ci/test_thrust.sh"
 
   cub:
-    name: CUB-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
@@ -53,7 +53,7 @@ jobs:
       test_script: "./ci/test_cub.sh"
   
   libcudacxx:
-    name: libcudacxx-CUDA${{ matrix.cuda_version }}-${{ matrix.compiler }}
+    name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:

From 5ea15cb4338b21a49b526ce2b2722f4c2ca5c565 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:08:24 -0500
Subject: [PATCH 258/395] Shorten name

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 89f54962a97..da2a7b8df9c 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -12,7 +12,7 @@ jobs:
   # ensures that the build/test steps can overlap across different configurations. For example,
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
-    #name: Build and test ${{matrix.compiler.name}}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
+    name: Build and test 
     uses: ./.github/workflows/build-and-test.yml
     strategy:
       fail-fast: false

From 629050feb787dc3defde541b7bc389352ef1d3b8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:14:43 -0500
Subject: [PATCH 259/395] Use cpu as name.

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index da2a7b8df9c..bc9f382195a 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -12,7 +12,7 @@ jobs:
   # ensures that the build/test steps can overlap across different configurations. For example,
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
-    name: Build and test 
+    name: ${{matrix.cpu}}
     uses: ./.github/workflows/build-and-test.yml
     strategy:
       fail-fast: false

From 88f65d02eb9203106cbd1a2a4895daf02c6d09f4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:38:29 -0500
Subject: [PATCH 260/395] Fix matrix passed for thrust

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 523f23eff79..439de30af7e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -34,7 +34,7 @@ jobs:
         cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
-      per_cuda_compiler_matrix: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 

From e235125ff74a93fbb52ae0aaf8fbda9d5b52d0bd Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:41:23 -0500
Subject: [PATCH 261/395] Remove CPU from leaf job name.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 37d10efe672..a6aa4223414 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -18,7 +18,7 @@ on:
 
 jobs:
   build:
-    name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
+    name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-cpu16
     #container: # Fetch the container image to use for the job

From fb4401d25610398b4f607430fa22836de330c792 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:48:24 -0500
Subject: [PATCH 262/395] Remove CPU from test name.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a6aa4223414..298c6fb040a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -46,7 +46,7 @@ jobs:
   test:
     needs: build
     if: ${{ needs.build.result == 'success' }}
-    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}/${{inputs.cpu}}
+    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
     #container: # Fetch the container image to use for the job

From e3c09ae828b62539775bf9e5e2c8ef3a15830e29 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 31 May 2023 09:49:08 -0500
Subject: [PATCH 263/395] Remove old test prints.

---
 .github/workflows/build-and-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 298c6fb040a..45a1fb6e402 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -37,7 +37,6 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
-          echo "test"
           if [[ -n "${{ inputs.build_script }}" ]]; then
             ./${{ inputs.build_script }}
           else
@@ -65,7 +64,6 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run test script
         run: | 
-          echo "test"
           if [[ -n "${{ inputs.test_script }}" ]]; then
             ./${{ inputs.test_script }}
           else

From 365fa2b461306c848f4263e020dd2d8abd0b3c4c Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 1 Jun 2023 12:19:12 -0400
Subject: [PATCH 264/395] Allow TOPLEVEL_PROJECT logic to be force enabled.

---
 CMakeLists.txt | 2 --
 cub            | 2 +-
 libcudacxx     | 2 +-
 thrust         | 2 +-
 4 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e40be2c4215..7adca5d5c4d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,8 +6,6 @@ cmake_minimum_required(VERSION 3.15)
 # another project via add_subdirectory()
 if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
   set(CCCL_TOPLEVEL_PROJECT ON)
-else()
-  set(CCCL_TOPLEVEL_PROJECT OFF)
 endif()
 
 # TODO find_package for add_subdir usecases, use components for subprojects.
diff --git a/cub b/cub
index 6477d389833..8c5cbe3ad7e 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 6477d389833321bc0ada92fedc3a5f8d76d6d0aa
+Subproject commit 8c5cbe3ad7e87248888cf2ee82c0088532cf0db9
diff --git a/libcudacxx b/libcudacxx
index c9a2d691c52..542397d6c74 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit c9a2d691c520bdbaa5783db74b08c2c51435d80d
+Subproject commit 542397d6c74191cc0421b74609e64efb96561724
diff --git a/thrust b/thrust
index 7740478ceaf..5469abd91cf 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 7740478ceaf375c34b8e1bc280d94493e710ab96
+Subproject commit 5469abd91cf450bb81ef55beaa9369a194b71d9e

From dc645485d4ed9c4b50ab00b11babf0f6603162d4 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 1 Jun 2023 12:20:14 -0400
Subject: [PATCH 265/395] Clean up some comments.

---
 CMakeLists.txt | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7adca5d5c4d..529f8f69a8e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,8 +8,6 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
   set(CCCL_TOPLEVEL_PROJECT ON)
 endif()
 
-# TODO find_package for add_subdir usecases, use components for subprojects.
-
 project(CCCL NONE)
 
 # Optionally include installation rules for non-top-level builds:
@@ -18,8 +16,7 @@ if (CCCL_ENABLE_INSTALL_RULES)
   include(cmake/CCCLInstallRules.cmake)
 endif()
 
-# Support adding Thrust to a parent project via add_subdirectory.
-# See examples/cmake/add_subdir/CMakeLists.txt for details.
+# Support adding CCCL to a parent project via add_subdirectory.
 if (NOT CCCL_TOPLEVEL_PROJECT)
   include(cmake/CCCLAddSubdir.cmake)
   return()

From fbf3bf96ad819a320b1a45e3d576ee8a680e05e6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 1 Jun 2023 14:17:36 -0500
Subject: [PATCH 266/395] Add Thrust and CUB build scripts.

---
 .github/workflows/build-and-test.yml |  6 ++-
 ci/build_cub.sh                      | 66 ++++++++++++++++----------
 ci/build_thrust.sh                   | 70 ++++++++++++++++++----------
 3 files changed, 92 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 45a1fb6e402..636069136d8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -31,6 +31,8 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
+          path: repo
+          submodules: 'recursive'
           fetch-depth: 0
           persist-credentials: false
       - name: Configure credentials and environment variables for sccache
@@ -38,7 +40,7 @@ jobs:
       - name: Run build script
         run: | 
           if [[ -n "${{ inputs.build_script }}" ]]; then
-            ./${{ inputs.build_script }}
+            time ./${{ inputs.build_script }}
           else
             echo "No build script provided, skipping."
           fi
@@ -65,7 +67,7 @@ jobs:
       - name: Run test script
         run: | 
           if [[ -n "${{ inputs.test_script }}" ]]; then
-            ./${{ inputs.test_script }}
+            time ./${{ inputs.test_script }}
           else
             echo "No test script provided, skipping."
           fi
diff --git a/ci/build_cub.sh b/ci/build_cub.sh
index 2782951996b..1836814c613 100755
--- a/ci/build_cub.sh
+++ b/ci/build_cub.sh
@@ -1,26 +1,44 @@
 #!/bin/bash
 
-echo "build cub"
-
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
-
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
-
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
-
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+echo "Begin CUB build"
+pwd
+
+# Check if the correct number of arguments has been provided
+if [ "$#" -ne 3 ]; then
+    echo "Usage: ./build_cub.sh <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
+    echo "The PARALLEL_LEVEL environment variable controls the amount of build parallelism. Default is the number of cores."
+    echo "Example: PARALLEL_LEVEL=8 ./build_cub.sh g++-8 14 \"70\" "
+    echo "Example: ./build_cub.sh clang++-8 17 \"70;75;80\" "
+    exit 1
+fi
+
+# Assign command line arguments to variables
+HOST_COMPILER=$1
+CXX_STANDARD=$2
+
+# Replace spaces, commas and semicolons with semicolons for CMake list
+GPU_ARCHS=$(echo $3 | tr ' ,' ';')
+
+PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
+
+cmake -S .. -B ../build \
+    -DCCCL_ENABLE_THRUST=OFF \
+    -DCCCL_ENABLE_LIBCUDACXX=OFF \
+    -DCCCL_ENABLE_CUB=ON \
+    -DCCCL_ENABLE_TESTING=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_CXX_COMPILER=${HOST_COMPILER} \
+    -DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS} \
+    -DCUB_ENABLE_DIALECT_CPP11=$(if [[ $CXX_STANDARD -ne 11 ]]; then echo "OFF"; else echo "ON"; fi) \
+    -DCUB_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
+    -DCUB_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
+    -DCUB_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
+    -G Ninja
+
+# Build the tests
+cmake --build ../build 
+
+echo "CUB build complete"
diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 4f901e064b4..fdfae6ead07 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,26 +1,48 @@
 #!/bin/bash
 
-echo "build thrust"
-
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
-
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
-
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
-
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+echo "Begin Thrust build"
+pwd
+
+# Check if the correct number of arguments has been provided
+if [ "$#" -ne 3 ]; then
+    echo "Usage: ./build_thrust.sh <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
+    echo "The PARALLEL_LEVEL environment variable controls the amount of build parallelism. Default is the number of cores."
+    echo "Example: PARALLEL_LEVEL=8 ./build_thrust.sh g++-8 14 \"70\" "
+    echo "Example: ./build_thrust.sh clang++-8 17 \"70;75;80\" "
+    exit 1
+fi
+
+# Assign command line arguments to variables
+HOST_COMPILER=$1
+CXX_STANDARD=$2
+
+# Replace spaces, commas and semicolons with semicolons for CMake list
+GPU_ARCHS=$(echo $3 | tr ' ,' ';')
+
+PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
+
+cmake -S .. -B ../build \
+      -DCCCL_ENABLE_THRUST=ON \
+      -DCCCL_ENABLE_LIBCUDACXX=OFF \
+      -DCCCL_ENABLE_CUB=OFF \
+      -DCCCL_ENABLE_TESTING=OFF \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_CXX_COMPILER=${HOST_COMPILER} \
+      -DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS} \
+      -DTHRUST_ENABLE_MULTICONFIG=ON \
+      -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=$(if [[ $CXX_STANDARD -ne 11 ]]; then echo "OFF"; else echo "ON"; fi) \
+      -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
+      -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
+      -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
+      -DTHRUST_MULTICONFIG_WORKLOAD=SMALL \
+      -G Ninja
+      # TODO: Add this back after Thrust removes the check against it.
+      #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \
+
+# Build the tests
+cmake --build ../build --parallel ${PARALLEL_LEVEL} 
+
+echo "Thrust build complete"

From ad2059fd69d1acf66d15bbbb87e4d49df3332978 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 1 Jun 2023 15:50:15 -0400
Subject: [PATCH 267/395] Move source-tree location of cccl CMake package to
 lib/.

---
 cmake/CCCLInstallRules.cmake                        | 2 +-
 {share => lib}/cmake/cccl/cccl-config-version.cmake | 0
 {share => lib}/cmake/cccl/cccl-config.cmake         | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename {share => lib}/cmake/cccl/cccl-config-version.cmake (100%)
 rename {share => lib}/cmake/cccl/cccl-config.cmake (100%)

diff --git a/cmake/CCCLInstallRules.cmake b/cmake/CCCLInstallRules.cmake
index 39c22d37a58..ca44b1b3d22 100644
--- a/cmake/CCCLInstallRules.cmake
+++ b/cmake/CCCLInstallRules.cmake
@@ -4,7 +4,7 @@ include(GNUInstallDirs)
 # CCCL has no installable binaries, no need to build before installing:
 set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)
 
-install(DIRECTORY "${CCCL_SOURCE_DIR}/share/cmake/cccl"
+install(DIRECTORY "${CCCL_SOURCE_DIR}/lib/cmake/cccl"
   DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/"
   PATTERN *.cmake.in EXCLUDE
 )
diff --git a/share/cmake/cccl/cccl-config-version.cmake b/lib/cmake/cccl/cccl-config-version.cmake
similarity index 100%
rename from share/cmake/cccl/cccl-config-version.cmake
rename to lib/cmake/cccl/cccl-config-version.cmake
diff --git a/share/cmake/cccl/cccl-config.cmake b/lib/cmake/cccl/cccl-config.cmake
similarity index 100%
rename from share/cmake/cccl/cccl-config.cmake
rename to lib/cmake/cccl/cccl-config.cmake

From d69ddf4f3d2ea66e72628ba58c4d748ff2734274 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 1 Jun 2023 15:58:04 -0400
Subject: [PATCH 268/395] Force exact version checks when locating CCCL
 subprojects.

---
 lib/cmake/cccl/cccl-config-version.cmake | 1 -
 lib/cmake/cccl/cccl-config.cmake         | 8 +++-----
 libcudacxx                               | 2 +-
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/cmake/cccl/cccl-config-version.cmake b/lib/cmake/cccl/cccl-config-version.cmake
index 97a42b140a1..1e026c98334 100644
--- a/lib/cmake/cccl/cccl-config-version.cmake
+++ b/lib/cmake/cccl/cccl-config-version.cmake
@@ -1,4 +1,3 @@
-# TODO How should we encode this as a source of truth?
 set(CCCL_VERSION_MAJOR 2)
 set(CCCL_VERSION_MINOR 2)
 set(CCCL_VERSION_PATCH 0)
diff --git a/lib/cmake/cccl/cccl-config.cmake b/lib/cmake/cccl/cccl-config.cmake
index b2e7e211085..bfed2d01452 100644
--- a/lib/cmake/cccl/cccl-config.cmake
+++ b/lib/cmake/cccl/cccl-config.cmake
@@ -4,8 +4,6 @@
 # Imports the Thrust, CUB, and libcudacxx components of the NVIDIA
 # CUDA/C++ Core Libraries.
 
-unset(cccl_version) # TODO
-
 set(cccl_cmake_dir "${CMAKE_CURRENT_LIST_DIR}")
 
 if (${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
@@ -21,7 +19,7 @@ foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
   endif()
 
   if(component STREQUAL "libcudacxx")
-    find_package(libcudacxx ${cccl_version} CONFIG
+    find_package(libcudacxx ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
       NO_DEFAULT_PATH # Only check the explicit HINTS below:
@@ -30,7 +28,7 @@ foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
         "${cccl_cmake_dir}/.."                             # Install layout
       )
   elseif(component STREQUAL "CUB")
-    find_package(CUB ${cccl_version} CONFIG
+    find_package(CUB ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
       NO_DEFAULT_PATH # Only check the explicit HINTS below:
@@ -39,7 +37,7 @@ foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
         "${cccl_cmake_dir}/.."                      # Install layout
     )
   elseif(component STREQUAL "Thrust")
-    find_package(Thrust ${cccl_version} CONFIG
+    find_package(Thrust ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
       NO_DEFAULT_PATH # Only check the explicit HINTS below:
diff --git a/libcudacxx b/libcudacxx
index 542397d6c74..652a99c4624 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit 542397d6c74191cc0421b74609e64efb96561724
+Subproject commit 652a99c4624291c2c6253565f88a62aea5d70265

From 21f63ef97eba460617c92ed7850b09b84092beeb Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 1 Jun 2023 15:31:40 -0500
Subject: [PATCH 269/395] Add devcontainers and generator script.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json | 21 ++++++++++++
 .devcontainer/cuda11.0-gcc8/devcontainer.json | 21 ++++++++++++
 .devcontainer/cuda11.0-gcc9/devcontainer.json | 21 ++++++++++++
 .../cuda11.0-llvm9/devcontainer.json          | 21 ++++++++++++
 .../cuda12.1-gcc10/devcontainer.json          | 21 ++++++++++++
 .../cuda12.1-gcc11/devcontainer.json          | 21 ++++++++++++
 .../cuda12.1-gcc12/devcontainer.json          | 21 ++++++++++++
 .devcontainer/cuda12.1-gcc7/devcontainer.json | 21 ++++++++++++
 .devcontainer/cuda12.1-gcc8/devcontainer.json | 21 ++++++++++++
 .devcontainer/cuda12.1-gcc9/devcontainer.json | 21 ++++++++++++
 .../cuda12.1-llvm10/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm11/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm12/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm13/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm14/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm15/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm16/devcontainer.json         | 21 ++++++++++++
 .../cuda12.1-llvm9/devcontainer.json          | 21 ++++++++++++
 .devcontainer/devcontainer_template.json      | 18 ++++++++++
 .devcontainer/make_devcontainers.sh           | 34 +++++++++++++++++++
 20 files changed, 430 insertions(+)
 create mode 100644 .devcontainer/cuda11.0-gcc7/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-gcc8/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-gcc9/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-llvm9/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc10/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc11/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc12/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc7/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc8/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc9/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm10/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm11/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm12/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm13/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm14/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm15/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm16/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm9/devcontainer.json
 create mode 100644 .devcontainer/devcontainer_template.json
 create mode 100755 .devcontainer/make_devcontainers.sh

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
new file mode 100644
index 00000000000..d4d4e4496a7
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc7/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04"
+}
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
new file mode 100644
index 00000000000..27fce6155d8
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc8/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04"
+}
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
new file mode 100644
index 00000000000..4dd2cf7a8f9
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc9/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04"
+}
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
new file mode 100644
index 00000000000..88ac9e2fa45
--- /dev/null
+++ b/.devcontainer/cuda11.0-llvm9/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
new file mode 100644
index 00000000000..1259e82a665
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
new file mode 100644
index 00000000000..788745f63cd
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
new file mode 100644
index 00000000000..cf8da0c355e
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
new file mode 100644
index 00000000000..97ce155de96
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
new file mode 100644
index 00000000000..a8160b76c84
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
new file mode 100644
index 00000000000..6c0bf3c5a5a
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
new file mode 100644
index 00000000000..ad8f8098c55
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
new file mode 100644
index 00000000000..a199f3599a7
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
new file mode 100644
index 00000000000..a8168f1dadd
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
new file mode 100644
index 00000000000..e4772275a17
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
new file mode 100644
index 00000000000..f228c21d04f
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
new file mode 100644
index 00000000000..d8a7dfe0cc3
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm16/devcontainer.json b/.devcontainer/cuda12.1-llvm16/devcontainer.json
new file mode 100644
index 00000000000..81b08a85da9
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm16/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04"
+}
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
new file mode 100644
index 00000000000..407f5231d82
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04"
+}
diff --git a/.devcontainer/devcontainer_template.json b/.devcontainer/devcontainer_template.json
new file mode 100644
index 00000000000..510748d8a25
--- /dev/null
+++ b/.devcontainer/devcontainer_template.json
@@ -0,0 +1,18 @@
+
+{
+  "hostRequirements": { "gpu": true },
+
+  "initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,config}"],
+
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ]
+}
\ No newline at end of file
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
new file mode 100755
index 00000000000..c337fc1f0ae
--- /dev/null
+++ b/.devcontainer/make_devcontainers.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+base_devcontainer_file="devcontainer_template.json"
+
+# Define image root
+IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"
+
+# Read matrix.yaml and convert it to json
+matrix_json=$(yq -o json ../ci/matrix.yaml)
+
+# Get unique combinations of cuda version, compiler name/version, and Ubuntu version
+combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, ubuntu: .ubuntu}] | unique | .[]')
+
+# For each unique combination
+for combination in $combinations; do
+    cuda_version=$(echo "$combination" | jq -r '.cuda')
+    compiler_name=$(echo "$combination" | jq -r '.compiler_name')
+    compiler_version=$(echo "$combination" | jq -r '.compiler_version')
+    ubuntu_version=$(echo "$combination" | jq -r '.ubuntu')
+
+    directory="cuda$cuda_version-$compiler_name$compiler_version"
+
+    mkdir -p "$directory"
+
+    devcontainer_file="$directory/devcontainer.json"
+
+    image_name="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
+
+    # Use the base_devcontainer.json as a template, plug in the CUDA, compiler names, versions, and Ubuntu version,
+    # and write the output to the new devcontainer.json file
+    jq --arg image_name "$image_name" '. + {image: $image_name}' $base_devcontainer_file > "$devcontainer_file"
+
+    echo "Created $devcontainer_file"
+done

From d7185e1a1885b83ae970977133b4139f59494e80 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 1 Jun 2023 15:50:59 -0500
Subject: [PATCH 270/395] Add a root devcontainer.json instead of a template.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json       |  4 ++--
 .devcontainer/cuda11.0-gcc8/devcontainer.json       |  4 ++--
 .devcontainer/cuda11.0-gcc9/devcontainer.json       |  4 ++--
 .devcontainer/cuda11.0-llvm9/devcontainer.json      |  4 ++--
 .devcontainer/cuda12.1-gcc10/devcontainer.json      |  4 ++--
 .devcontainer/cuda12.1-gcc11/devcontainer.json      |  4 ++--
 .devcontainer/cuda12.1-gcc12/devcontainer.json      |  4 ++--
 .devcontainer/cuda12.1-gcc7/devcontainer.json       |  4 ++--
 .devcontainer/cuda12.1-gcc8/devcontainer.json       |  4 ++--
 .devcontainer/cuda12.1-gcc9/devcontainer.json       |  4 ++--
 .devcontainer/cuda12.1-llvm10/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm11/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm12/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm13/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm14/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm15/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm16/devcontainer.json     |  4 ++--
 .devcontainer/cuda12.1-llvm9/devcontainer.json      |  4 ++--
 ...devcontainer_template.json => devcontainer.json} |  2 ++
 .devcontainer/make_devcontainers.sh                 | 13 ++++++++++++-
 20 files changed, 50 insertions(+), 37 deletions(-)
 rename .devcontainer/{devcontainer_template.json => devcontainer.json} (88%)

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
index d4d4e4496a7..e835028cb5e 100644
--- a/.devcontainer/cuda11.0-gcc7/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc7/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04"
+  ]
 }
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
index 27fce6155d8..4fcd549d4cd 100644
--- a/.devcontainer/cuda11.0-gcc8/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc8/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04"
+  ]
 }
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
index 4dd2cf7a8f9..ab422f02624 100644
--- a/.devcontainer/cuda11.0-gcc9/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04"
+  ]
 }
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
index 88ac9e2fa45..d377c2e1ba9 100644
--- a/.devcontainer/cuda11.0-llvm9/devcontainer.json
+++ b/.devcontainer/cuda11.0-llvm9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
index 1259e82a665..7795501421a 100644
--- a/.devcontainer/cuda12.1-gcc10/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
index 788745f63cd..33ce4225b41 100644
--- a/.devcontainer/cuda12.1-gcc11/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
index cf8da0c355e..cb518a231cc 100644
--- a/.devcontainer/cuda12.1-gcc12/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
index 97ce155de96..4372c2333e2 100644
--- a/.devcontainer/cuda12.1-gcc7/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
index a8160b76c84..1c2171a821b 100644
--- a/.devcontainer/cuda12.1-gcc8/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
index 6c0bf3c5a5a..72ecb726225 100644
--- a/.devcontainer/cuda12.1-gcc9/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
index ad8f8098c55..bd541afd5ec 100644
--- a/.devcontainer/cuda12.1-llvm10/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
index a199f3599a7..292c2138b41 100644
--- a/.devcontainer/cuda12.1-llvm11/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
index a8168f1dadd..55caa622f8d 100644
--- a/.devcontainer/cuda12.1-llvm12/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
index e4772275a17..4b7792580cd 100644
--- a/.devcontainer/cuda12.1-llvm13/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
index f228c21d04f..15b905b1508 100644
--- a/.devcontainer/cuda12.1-llvm14/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
index d8a7dfe0cc3..3a18b9a8843 100644
--- a/.devcontainer/cuda12.1-llvm15/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm16/devcontainer.json b/.devcontainer/cuda12.1-llvm16/devcontainer.json
index 81b08a85da9..d020ee58aa3 100644
--- a/.devcontainer/cuda12.1-llvm16/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm16/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04"
+  ]
 }
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
index 407f5231d82..ae74bd6de22 100644
--- a/.devcontainer/cuda12.1-llvm9/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
   },
@@ -16,6 +17,5 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04"
+  ]
 }
diff --git a/.devcontainer/devcontainer_template.json b/.devcontainer/devcontainer.json
similarity index 88%
rename from .devcontainer/devcontainer_template.json
rename to .devcontainer/devcontainer.json
index 510748d8a25..b60dd7cfc78 100644
--- a/.devcontainer/devcontainer_template.json
+++ b/.devcontainer/devcontainer.json
@@ -1,5 +1,7 @@
 
 {
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
+
   "hostRequirements": { "gpu": true },
 
   "initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,config}"],
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
index c337fc1f0ae..dde0d9a1c04 100755
--- a/.devcontainer/make_devcontainers.sh
+++ b/.devcontainer/make_devcontainers.sh
@@ -1,6 +1,17 @@
 #!/bin/bash
 
-base_devcontainer_file="devcontainer_template.json"
+# This script parses the CI matrix.yaml file and generates a devcontainer.json file for each unique combination of 
+# CUDA version, compiler name/version, and Ubuntu version. The devcontainer.json files are written to the
+# .devcontainer directory to a subdirectory named after the CUDA version and compiler name/version.
+# GitHub docs on using multiple devcontainer.json files:
+# https://docs.github.com/en/codespaces/setting-up-your-project-for-codespaces/adding-a-dev-container-configuration/introduction-to-dev-containers#devcontainerjson
+
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+# The root devcontainer.json file is used as a template for all other devcontainer.json files
+# by replacing the `image:` field with the appropriate image name
+base_devcontainer_file="./devcontainer.json"
 
 # Define image root
 IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"

From ad5556949059397d0734a19dbd3fa48d7150db73 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 1 Jun 2023 17:05:27 -0400
Subject: [PATCH 271/395] Fix various issues with CMake package testing.

- Support case-insensitive components for find_package(CCCL)
- Use cache vars to access libcudacxx version in validation test.
- Remove unnecessary `find_package` calls that were masking test issues.
---
 cub                                   |  2 +-
 lib/cmake/cccl/cccl-config.cmake      | 19 ++++++++++++++-----
 test/cmake/test_export/CMakeLists.txt |  9 +++------
 thrust                                |  2 +-
 4 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/cub b/cub
index 8c5cbe3ad7e..7bfe37a4a05 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 8c5cbe3ad7e87248888cf2ee82c0088532cf0db9
+Subproject commit 7bfe37a4a050bee834e325190bac0fb893746e7a
diff --git a/lib/cmake/cccl/cccl-config.cmake b/lib/cmake/cccl/cccl-config.cmake
index bfed2d01452..7bc6cf7b47e 100644
--- a/lib/cmake/cccl/cccl-config.cmake
+++ b/lib/cmake/cccl/cccl-config.cmake
@@ -12,13 +12,22 @@ else()
   set(cccl_quiet_flag "")
 endif()
 
-foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
+if (DEFINED ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS AND
+    ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS)
+  set(components ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
+else()
+  set(components Thrust CUB libcudacxx)
+endif()
+
+foreach(component IN LISTS components)
+  string(TOLOWER "${component}" component_lower)
+
   unset(req)
   if (${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED_${component})
     set(cccl_comp_required_flag "REQUIRED")
   endif()
 
-  if(component STREQUAL "libcudacxx")
+  if(component_lower STREQUAL "libcudacxx")
     find_package(libcudacxx ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
@@ -26,8 +35,8 @@ foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
       HINTS
         "${cccl_cmake_dir}/../../../libcudacxx/lib/cmake/" # Source layout (GitHub)
         "${cccl_cmake_dir}/.."                             # Install layout
-      )
-  elseif(component STREQUAL "CUB")
+    )
+  elseif(component_lower STREQUAL "cub")
     find_package(CUB ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
@@ -36,7 +45,7 @@ foreach(component ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS})
         "${cccl_cmake_dir}/../../../cub/cub/cmake/" # Source layout (GitHub)
         "${cccl_cmake_dir}/.."                      # Install layout
     )
-  elseif(component STREQUAL "Thrust")
+  elseif(component_lower STREQUAL "thrust")
     find_package(Thrust ${CCCL_VERSION} EXACT CONFIG
       ${cccl_quiet_flag}
       ${cccl_comp_required_flag}
diff --git a/test/cmake/test_export/CMakeLists.txt b/test/cmake/test_export/CMakeLists.txt
index 56e3ffb7f11..b10ea675430 100644
--- a/test/cmake/test_export/CMakeLists.txt
+++ b/test/cmake/test_export/CMakeLists.txt
@@ -105,18 +105,16 @@ foreach (component IN LISTS COMPONENTS)
   add_test(NAME ${test_target} COMMAND ${test_target})
 
   if (component STREQUAL "libcudacxx")
-    find_package(libcudacxx) # Make sure version vars are in scope
     # TODO Should this be a genex? Will these stay correct as versions change?
     math(EXPR component_cmake_version
-      "(${libcudacxx_VERSION_MAJOR} * 1000000) +
-        ${libcudacxx_VERSION_MINOR} * 1000 +
-        ${libcudacxx_VERSION_PATCH}")
+      "(${LIBCUDACXX_VERSION_MAJOR} * 1000000) +
+        ${LIBCUDACXX_VERSION_MINOR} * 1000 +
+        ${LIBCUDACXX_VERSION_PATCH}")
     target_compile_definitions(${test_target} PRIVATE
       "VERSION_HEADER=cuda/std/version"
       "VERSION_MACRO=_LIBCUDACXX_CUDA_API_VERSION"
       "EXPECTED_VERSION=${component_cmake_version}")
   elseif (component STREQUAL "CUB")
-    find_package(CUB) # Make sure version vars are in scope
     # TODO Should this be a genex? Will these stay correct as versions change?
     math(EXPR component_cmake_version
       "(${CUB_VERSION_MAJOR} * 100000) +
@@ -127,7 +125,6 @@ foreach (component IN LISTS COMPONENTS)
       "VERSION_MACRO=CUB_VERSION"
       "EXPECTED_VERSION=${component_cmake_version}")
   elseif (component STREQUAL "Thrust")
-    find_package(Thrust) # Make sure version vars are in scope
     # TODO Should this be a genex? Will these stay correct as versions change?
     math(EXPR component_cmake_version
       "(${THRUST_VERSION_MAJOR} * 100000) +
diff --git a/thrust b/thrust
index 5469abd91cf..7f9259c9738 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 5469abd91cf450bb81ef55beaa9369a194b71d9e
+Subproject commit 7f9259c973810460f68c9b0e3d39b95c387f2407

From 3eb36b0188e96aca20ebc906ea6732ebb7701976 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 13:00:07 +0000
Subject: [PATCH 272/395] Add test after checkout

---
 .github/workflows/build-and-test.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 636069136d8..8fa67559026 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -35,6 +35,11 @@ jobs:
           submodules: 'recursive'
           fetch-depth: 0
           persist-credentials: false
+      - name: pwd
+        run: |
+          pwd
+          ls
+          ls cccl
       - name: Configure credentials and environment variables for sccache
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script

From 5267ea801bfdb2f03f5784204143418479d0b6d7 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 13:07:11 +0000
Subject: [PATCH 273/395] Forgot that I'm checking out to repo/ now.

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8fa67559026..bbd0c01d9d5 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -39,9 +39,9 @@ jobs:
         run: |
           pwd
           ls
-          ls cccl
+          ls repo
       - name: Configure credentials and environment variables for sccache
-        uses: ./.github/actions/configure_cccl_sccache
+        uses: ./repo/.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
           if [[ -n "${{ inputs.build_script }}" ]]; then

From 4eadcac2ba62a6dd839198bcfb694e18fb182ca9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:15:57 +0000
Subject: [PATCH 274/395] Make build/test jobs conditional on input script.

---
 .github/workflows/build-and-test.yml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index bbd0c01d9d5..6a9fdf6e319 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -18,6 +18,7 @@ on:
 
 jobs:
   build:
+    if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-cpu16
@@ -44,14 +45,13 @@ jobs:
         uses: ./repo/.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
-          if [[ -n "${{ inputs.build_script }}" ]]; then
             time ./${{ inputs.build_script }}
-          else
-            echo "No build script provided, skipping."
-          fi
   test:
     needs: build
-    if: ${{ needs.build.result == 'success' }}
+    if: |
+      '!cancelled()' &&
+      ( needs.build.result == 'success' || needs.build.result == 'skipped') &&
+      inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
@@ -71,8 +71,4 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run test script
         run: | 
-          if [[ -n "${{ inputs.test_script }}" ]]; then
             time ./${{ inputs.test_script }}
-          else
-            echo "No test script provided, skipping."
-          fi

From 8fe63266954bef1eb2277e697b8a02c7a9daa7ba Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:21:16 +0000
Subject: [PATCH 275/395] Test only passing test script.

---
 .github/workflows/pr.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 439de30af7e..e08b4f8afc8 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -35,8 +35,8 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
+      build_script: #"./ci/build_thrust.sh"
+      test_script: #"./ci/test_thrust.sh"
 
   cub:
     name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -49,8 +49,8 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
+      build_script: #"./ci/build_cub.sh"
+      test_script: #"./ci/test_cub.sh"
   
   libcudacxx:
     name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -63,5 +63,5 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: "./ci/build_libcudacxx.sh"
+      build_script: #"./ci/build_libcudacxx.sh"
       test_script: "./ci/test_libcudacxx.sh" 

From 52196bbc72c09d63e8b029c82cf5dbbdb7955f0f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:26:43 +0000
Subject: [PATCH 276/395] Rename "build_type" to "matrix_type" to avoid
 confusion.

---
 .github/workflows/compute-matrix.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
index 638b688d3a9..bbd09e628f5 100644
--- a/.github/workflows/compute-matrix.yml
+++ b/.github/workflows/compute-matrix.yml
@@ -1,5 +1,5 @@
 
-name: Compute the matrix for a given build type from the specified matrix file
+name: Compute the matrix for a given matrix type from the specified matrix file
 
 on:
   workflow_call:
@@ -7,13 +7,13 @@ on:
       matrix_file: 
         type: string
         required: true
-      build_type:
-        description: "The desired build type. Currently, only 'pull-request' and 'nightly' are supported."
+      matrix_type:
+        description: "The desired matrix type. Currently, only 'pull-request' and 'nightly' are supported."
         type: string
         required: true
     outputs:
       FULL_MATRIX: 
-        description: "The full matrix of build configurations for the given build type."
+        description: "The full matrix of build configurations for the given matrix type."
         value: ${{ jobs.compute-matrix.outputs.FULL_MATRIX }}
       CUDA_VERSIONS:
         description: "The unique CUDA versions in the full matrix."
@@ -30,12 +30,12 @@ defaults:
     shell: bash
 
 env:
-  BUILD_TYPE: ${{ inputs.build_type}}
+  MATRIX_TYPE: ${{ inputs.matrix_type}}
   MATRIX_FILE: ${{ inputs.matrix_file }}
 
 jobs:
   compute-matrix:
-    name: Compute ${{inputs.build_type}} matrix
+    name: ${{inputs.matrix_type}} 
     runs-on: ubuntu-latest
     outputs:
       FULL_MATRIX:  ${{ steps.compute-matrix.outputs.FULL_MATRIX }}
@@ -43,10 +43,10 @@ jobs:
       COMPILERS: ${{ steps.compute-matrix.outputs.COMPILERS }}
       PER_CUDA_COMPILER_MATRIX: ${{ steps.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX }}
     steps:
-      - name: Validate build type
+      - name: Validate matrix type
         run: |
-          if [[ "$BUILD_TYPE" != "pull-request" && "$BUILD_TYPE" != "nightly" ]]; then
-            echo "Invalid build type: $BUILD_TYPE. Must be 'nightly' or 'pull-request'"
+          if [[ "$MATRIX_TYPE" != "pull-request" && "$MATRIX_TYPE" != "nightly" ]]; then
+            echo "Invalid build type: $MATRIX_TYPE. Must be 'nightly' or 'pull-request'"
             exit 1
           fi
       - name: Checkout repo
@@ -56,7 +56,7 @@ jobs:
         run: |
           echo "Input matrix file:"
           cat "$MATRIX_FILE"
-          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg build_type "$BUILD_TYPE" '[ .[$build_type][] | . as $o | {std: .std[]} + del($o.std)]')
+          FULL_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c --arg matrix_type "$MATRIX_TYPE" '[ .[$matrix_type][] | . as $o | {std: .std[]} + del($o.std)]')
           echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT"
           CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
           echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"

From 9b7700d8fd271e39040fab5c3666d19a175716ae Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:28:51 +0000
Subject: [PATCH 277/395] Try "always()" instead of !cancelled().

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6a9fdf6e319..f43d1c0959a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -49,7 +49,7 @@ jobs:
   test:
     needs: build
     if: |
-      '!cancelled()' &&
+      always() &&
       ( needs.build.result == 'success' || needs.build.result == 'skipped') &&
       inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}

From 4949bb06c5c7db997104ffc1e2663495510613f6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:29:48 +0000
Subject: [PATCH 278/395] build_type -> matrix_type

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index e08b4f8afc8..c9354459fed 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -22,7 +22,7 @@ jobs:
     uses: ./.github/workflows/compute-matrix.yml
     with:
       matrix_file: "./ci/matrix.yaml"
-      build_type: "pull-request"
+      matrix_type: "pull-request"
 
   thrust:
     name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}

From 9351ecc0001af16395f018e1be1581e5f39557e1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:36:12 +0000
Subject: [PATCH 279/395] Try !cancelled().

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f43d1c0959a..6a9fdf6e319 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -49,7 +49,7 @@ jobs:
   test:
     needs: build
     if: |
-      always() &&
+      '!cancelled()' &&
       ( needs.build.result == 'success' || needs.build.result == 'skipped') &&
       inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}

From 5dd0b0a94f3317cfb0e40af904111f91be88f2ab Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:39:23 +0000
Subject: [PATCH 280/395] Try committing again?

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6a9fdf6e319..17bc033da31 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,7 +50,7 @@ jobs:
     needs: build
     if: |
       '!cancelled()' &&
-      ( needs.build.result == 'success' || needs.build.result == 'skipped') &&
+      ( needs.build.result == 'success' || needs.build.result == 'skipped' ) &&
       inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest

From c2adb0b603ab7f9900e2ca7d8ddfce3ac834ff55 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:46:31 +0000
Subject: [PATCH 281/395] always()

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 17bc033da31..1d6f3485ced 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -49,7 +49,7 @@ jobs:
   test:
     needs: build
     if: |
-      '!cancelled()' &&
+      always() &&
       ( needs.build.result == 'success' || needs.build.result == 'skipped' ) &&
       inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}

From d7f2372b896c634f6ee9c5554d462ef2cd50ed73 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 14:58:16 +0000
Subject: [PATCH 282/395] success() || failure()

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 1d6f3485ced..d98e26655d8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -49,7 +49,7 @@ jobs:
   test:
     needs: build
     if: |
-      always() &&
+      ( success() || failure() ) &&
       ( needs.build.result == 'success' || needs.build.result == 'skipped' ) &&
       inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}

From d2f743bfbcb50e72408df61595b2b0b641f17caa Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 15:06:54 +0000
Subject: [PATCH 283/395] Try it on a single line?

---
 .github/workflows/build-and-test.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d98e26655d8..0fe584e7cb3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -48,10 +48,7 @@ jobs:
             time ./${{ inputs.build_script }}
   test:
     needs: build
-    if: |
-      ( success() || failure() ) &&
-      ( needs.build.result == 'success' || needs.build.result == 'skipped' ) &&
-      inputs.test_script != ''
+    if:  ( success() || failure() ) && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != ''
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1

From 0c53f87956a89c3a3abecb30a82d5a8f5d403e3b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 15:17:37 +0000
Subject: [PATCH 284/395] Use expression syntax

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 0fe584e7cb3..e6717351463 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -48,7 +48,7 @@ jobs:
             time ./${{ inputs.build_script }}
   test:
     needs: build
-    if:  ( success() || failure() ) && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != ''
+    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ubuntu-latest
     #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1

From 8f1bf448b7df4bf5d5cbbfbc6ef245d99e3159eb Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 16:06:48 +0000
Subject: [PATCH 285/395] Revert checking out into repo folder for now.

---
 .github/workflows/build-and-test.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index e6717351463..09767246ce1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -32,17 +32,11 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
-          path: repo
           submodules: 'recursive'
           fetch-depth: 0
           persist-credentials: false
-      - name: pwd
-        run: |
-          pwd
-          ls
-          ls repo
       - name: Configure credentials and environment variables for sccache
-        uses: ./repo/.github/actions/configure_cccl_sccache
+        uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
             time ./${{ inputs.build_script }}

From 57b773804ad89ac7cf4e9be67136e2a93e547242 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:22:43 +0000
Subject: [PATCH 286/395] Ensure we get full path to host compiler in build
 script.

---
 ci/build_thrust.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index fdfae6ead07..6d4b95bc464 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -3,8 +3,6 @@
 # Ensure the script is being executed in its containing directory
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
-echo "Begin Thrust build"
-pwd
 
 # Check if the correct number of arguments has been provided
 if [ "$#" -ne 3 ]; then
@@ -16,7 +14,7 @@ if [ "$#" -ne 3 ]; then
 fi
 
 # Assign command line arguments to variables
-HOST_COMPILER=$1
+HOST_COMPILER=$(which $1)
 CXX_STANDARD=$2
 
 # Replace spaces, commas and semicolons with semicolons for CMake list
@@ -24,6 +22,15 @@ GPU_ARCHS=$(echo $3 | tr ' ,' ';')
 
 PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
 
+echo "========================================"
+echo "Begin Thrust build"
+echo "pwd=$(pwd)"
+echo "HOST_COMPILER=$HOST_COMPILER"
+echo "CXX_STANDARD=$CXX_STANDARD"
+echo "GPU_ARCHS=$GPU_ARCHS"
+echo "PARALLEL_LEVEL=$PARALLEL_LEVEL"
+echo "========================================"
+
 cmake -S .. -B ../build \
       -DCCCL_ENABLE_THRUST=ON \
       -DCCCL_ENABLE_LIBCUDACXX=OFF \

From 47af73a5be2121b5772b412dc9c74995509c81a9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:29:18 +0000
Subject: [PATCH 287/395] Enable running thrust build on actual container.

---
 .github/workflows/build-and-test.yml          | 15 +++++++++------
 .github/workflows/dispatch-build-and-test.yml |  2 ++
 .github/workflows/pr.yml                      |  2 +-
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 09767246ce1..bf472172261 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -9,10 +9,12 @@ on:
     inputs:
       cuda_version: {type: string, required: true}
       compiler: {type: string, required: true}
+      compiler_exe: {type: string, required: true}
       compiler_version: {type: string, required: true}
       std: {type: string, required: true}
       gpu_build_archs: {type: string, required: true}
       cpu: {type: string, required: true}
+      ubuntu: {type: string, required: true}
       build_script: {type: string, required: false}
       test_script: {type: string, required: false }
 
@@ -20,11 +22,10 @@ jobs:
   build:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: ubuntu-latest
-    #runs-on: linux-${{inputs.cpu}}-cpu16
-    #container: # Fetch the container image to use for the job
-    #  options: -u root
-    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+    runs-on: linux-${{inputs.cpu}}-cpu16
+    container: 
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-ubuntu${{ inputs.ubuntu }}
     permissions:
       id-token: write
       contents: read
@@ -39,7 +40,9 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
-            time ./${{ inputs.build_script }}
+            sccache -s
+            time ./${{ inputs.build_script }} ${{inputs.compiler_exe}} ${{inputs.std}} ${{inputs.gpu_build_archs}}}
+            sccache -s
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index bc9f382195a..40d49119a7a 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -21,10 +21,12 @@ jobs:
     with:
       cuda_version: ${{ matrix.cuda }}
       compiler: ${{ matrix.compiler.name }}
+      compiler_exe: ${{ matrix.compiler.exe }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
       gpu_build_archs: "70" #TODO: Fix this
       cpu: ${{ matrix.cpu }}
+      ubunutu: ${{ matrix.ubuntu }}
       build_script: ${{ inputs.build_script }}
       test_script: ${{ inputs.test_script }}
 
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index c9354459fed..469b4ff9aa0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -35,7 +35,7 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: #"./ci/build_thrust.sh"
+      build_script: "./ci/build_thrust.sh"
       test_script: #"./ci/test_thrust.sh"
 
   cub:

From 9c1dfaad8e327e2814e47135441fc3aeab1d02f8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:30:29 +0000
Subject: [PATCH 288/395] Ubuntu typo.

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 40d49119a7a..d4c8be77611 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -26,7 +26,7 @@ jobs:
       std: ${{ matrix.std }}
       gpu_build_archs: "70" #TODO: Fix this
       cpu: ${{ matrix.cpu }}
-      ubunutu: ${{ matrix.ubuntu }}
+      ubuntu: ${{ matrix.ubuntu }}
       build_script: ${{ inputs.build_script }}
       test_script: ${{ inputs.test_script }}
 

From e816b827288f89d09902a5a23351f681782bc7ce Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:37:42 +0000
Subject: [PATCH 289/395] Remove fetch-depth 0 and persist-credentials.

---
 .github/workflows/build-and-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index bf472172261..2ec4e4f4d8b 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -34,8 +34,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: 'recursive'
-          fetch-depth: 0
-          persist-credentials: false
       - name: Configure credentials and environment variables for sccache
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script

From 0158c3b9790122585391ab44841f54bb89001db6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:39:46 +0000
Subject: [PATCH 290/395] Disable cub/libcu++ for now.

---
 .github/workflows/pr.yml | 52 ++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 469b4ff9aa0..1f3168c636f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -38,30 +38,30 @@ jobs:
       build_script: "./ci/build_thrust.sh"
       test_script: #"./ci/test_thrust.sh"
 
-  cub:
-    name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: #"./ci/build_cub.sh"
-      test_script: #"./ci/test_cub.sh"
+  #cub:
+  #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
+  #  needs: compute-matrix
+  #  uses: ./.github/workflows/dispatch-build-and-test.yml
+  #  strategy:
+  #    fail-fast: false
+  #    matrix: 
+  #      cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+  #      compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+  #  with:
+  #    per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+  #    build_script: #"./ci/build_cub.sh"
+  #    test_script: #"./ci/test_cub.sh"
   
-  libcudacxx:
-    name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-    needs: compute-matrix
-    uses: ./.github/workflows/dispatch-build-and-test.yml
-    strategy:
-      fail-fast: false
-      matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-    with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_script: #"./ci/build_libcudacxx.sh"
-      test_script: "./ci/test_libcudacxx.sh" 
+  #libcudacxx:
+  #  name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
+  #  needs: compute-matrix
+  #  uses: ./.github/workflows/dispatch-build-and-test.yml
+  #  strategy:
+  #    fail-fast: false
+  #    matrix: 
+  #      cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+  #      compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+  #  with:
+  #    per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+  #    build_script: #"./ci/build_libcudacxx.sh"
+  #    test_script: "./ci/test_libcudacxx.sh" 

From 26d952822077ddb58cb777d8dd84823928897199 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:43:00 +0000
Subject: [PATCH 291/395] echo git version in container

---
 .github/workflows/build-and-test.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2ec4e4f4d8b..737275546e5 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -30,6 +30,8 @@ jobs:
       id-token: write
       contents: read
     steps:
+      - name: Echo git version
+        run: git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:

From bdd73dc4d07abfbc5cebfbdcd772e344aaa39090 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:47:23 +0000
Subject: [PATCH 292/395] Update git for submodule support with action
 checkout.

---
 .github/workflows/build-and-test.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 737275546e5..c832746d452 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -32,6 +32,12 @@ jobs:
     steps:
       - name: Echo git version
         run: git --version
+      - name: Update git
+        run: | 
+          sudo add-apt-repository -y ppa:git-core/ppa
+          sudo apt-get update
+          sudo apt-get install -y git
+          git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:

From 5f30e95cb0dd0d0813f9589989dd09537153ede0 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:51:38 +0000
Subject: [PATCH 293/395] Update default shell to exit on first error.

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index c832746d452..00b3f58074c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -2,7 +2,7 @@ name: build and test
 
 defaults:
   run:
-    shell: bash
+    shell: bash -eo pipefail {0}
 
 on:
   workflow_call:
@@ -47,7 +47,7 @@ jobs:
       - name: Run build script
         run: | 
             sccache -s
-            time ./${{ inputs.build_script }} ${{inputs.compiler_exe}} ${{inputs.std}} ${{inputs.gpu_build_archs}}}
+            time ./${{ inputs.build_script }} ${{inputs.compiler_exe}} ${{inputs.std}} ${{inputs.gpu_build_archs}}
             sccache -s
   test:
     needs: build

From 84bdaa5d75c136549907d8c8c465f753fdb06368 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 17:54:50 +0000
Subject: [PATCH 294/395] Ignore C++ dialect deprecation warning.

---
 ci/build_thrust.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 6d4b95bc464..35af948165e 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -45,6 +45,7 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_WORKLOAD=SMALL \
+      -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \

From 76c4455c0804ff3cf6a6423672496c1122509e7c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 18:04:19 +0000
Subject: [PATCH 295/395] Padd through macro to ignore deprecation warnings.

---
 ci/build_thrust.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 35af948165e..0526ab9e4c2 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -44,7 +44,8 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
-      -DTHRUST_MULTICONFIG_WORKLOAD=SMALL \
+      -DCMAKE_CXX_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
+      -DCMAKE_CUDA_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
       -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.

From 5790bd73fdd7d42e1d17d20b509f2ae2f54bcc83 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 18:07:50 +0000
Subject: [PATCH 296/395] Set bash script to fail if any command fails.

---
 ci/build_thrust.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 0526ab9e4c2..fe688391576 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -xeuo pipefail
+
 # Ensure the script is being executed in its containing directory
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 

From 3656d1adfed0dd39f2b894f0b6e44f34d9e58250 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 18:08:43 +0000
Subject: [PATCH 297/395] Remove erroneous cmake flag

---
 ci/build_thrust.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index fe688391576..123a0718f90 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -48,7 +48,6 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DCMAKE_CXX_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
       -DCMAKE_CUDA_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
-      -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \

From 9f2a4c15506f5638df4060f1f14bd3d2528c7363 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 18:24:32 +0000
Subject: [PATCH 298/395] Ignore CUB deprecation warnings too.

---
 ci/build_thrust.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 123a0718f90..c5f11fb5401 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -46,8 +46,8 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
-      -DCMAKE_CXX_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
-      -DCMAKE_CUDA_FLAGS=-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT \
+      -DCMAKE_CXX_FLAGS="-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT -DCUB_IGNORE_DEPRECATED_CPP_DIALECT" \
+      -DCMAKE_CUDA_FLAGS="-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT -DCUB_IGNORE_DEPRECATED_CPP_DIALECT" \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \

From 06ba15d57565621b11bea9d7dbe35d45d20ff531 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 18:30:02 +0000
Subject: [PATCH 299/395] clang16 isn't supported.

---
 ci/matrix.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index a68c945a010..6fb8796706d 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -39,4 +39,3 @@ pull-request:
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
 - {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
 - {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '16', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}

From 5ddceb91d7ffe66a709cb848d4742ca8f55a8e16 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Fri, 2 Jun 2023 15:32:14 -0400
Subject: [PATCH 300/395] Bump libcudacxx to address review comments.

---
 libcudacxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcudacxx b/libcudacxx
index 652a99c4624..f6a0035d206 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit 652a99c4624291c2c6253565f88a62aea5d70265
+Subproject commit f6a0035d2069684ee503a12f3594db545dd8d03d

From 6d76b843bcc428cec568df1c24bd593154af8ba5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 19:38:21 +0000
Subject: [PATCH 301/395] Make build verbose.

---
 ci/build_thrust.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index c5f11fb5401..9505a552103 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -53,6 +53,6 @@ cmake -S .. -B ../build \
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \
 
 # Build the tests
-cmake --build ../build --parallel ${PARALLEL_LEVEL} 
+cmake --build ../build --parallel ${PARALLEL_LEVEL} -- -v
 
 echo "Thrust build complete"

From 4bdc3ea0dc507494f67245208c0b738878190c59 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Fri, 2 Jun 2023 20:22:05 +0000
Subject: [PATCH 302/395] Pass correct GPU archs through to build script

---
 .github/workflows/build-and-test.yml          |  2 +-
 .github/workflows/dispatch-build-and-test.yml |  2 +-
 ci/matrix.yaml                                | 34 +++++++++----------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 00b3f58074c..56ed75b4056 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -47,7 +47,7 @@ jobs:
       - name: Run build script
         run: | 
             sccache -s
-            time ./${{ inputs.build_script }} ${{inputs.compiler_exe}} ${{inputs.std}} ${{inputs.gpu_build_archs}}
+            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
             sccache -s
   test:
     needs: build
diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index d4c8be77611..05a493b3352 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -24,7 +24,7 @@ jobs:
       compiler_exe: ${{ matrix.compiler.exe }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
-      gpu_build_archs: "70" #TODO: Fix this
+      gpu_build_archs: ${{ matrix.gpu_build_archs }} 
       cpu: ${{ matrix.cpu }}
       ubuntu: ${{ matrix.ubuntu }}
       build_script: ${{ inputs.build_script }}
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 6fb8796706d..63a320f0833 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,20 +22,20 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: ['70', '80'], std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}

From adc8b08f752ba0c9e29cc4e98f710fedaaf52f6c Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Fri, 2 Jun 2023 22:10:50 -0400
Subject: [PATCH 303/395] Bump Thrust for RDC flag fix.

---
 thrust | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thrust b/thrust
index 7f9259c9738..572c19cef64 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 7f9259c973810460f68c9b0e3d39b95c387f2407
+Subproject commit 572c19cef64b3783c55ddbd60fc45106ed36d251

From 007cdebe88fa8a5508970e2c353c9f948204e2b1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 14:59:36 +0000
Subject: [PATCH 304/395] Add name field to devcontainers.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json   |  3 ++-
 .devcontainer/cuda11.0-gcc8/devcontainer.json   |  3 ++-
 .devcontainer/cuda11.0-gcc9/devcontainer.json   |  3 ++-
 .devcontainer/cuda11.0-llvm9/devcontainer.json  |  3 ++-
 .devcontainer/cuda12.1-gcc10/devcontainer.json  |  3 ++-
 .devcontainer/cuda12.1-gcc11/devcontainer.json  |  3 ++-
 .devcontainer/cuda12.1-gcc12/devcontainer.json  |  3 ++-
 .devcontainer/cuda12.1-gcc7/devcontainer.json   |  3 ++-
 .devcontainer/cuda12.1-gcc8/devcontainer.json   |  3 ++-
 .devcontainer/cuda12.1-gcc9/devcontainer.json   |  3 ++-
 .devcontainer/cuda12.1-llvm10/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm11/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm12/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm13/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm14/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm15/devcontainer.json |  3 ++-
 .devcontainer/cuda12.1-llvm9/devcontainer.json  |  3 ++-
 .devcontainer/make_devcontainers.sh             | 13 +++++--------
 18 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
index e835028cb5e..976ad3d8666 100644
--- a/.devcontainer/cuda11.0-gcc7/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc7/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda11.0-gcc7"
 }
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
index 4fcd549d4cd..ccf2707b6c6 100644
--- a/.devcontainer/cuda11.0-gcc8/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc8/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda11.0-gcc8"
 }
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
index ab422f02624..8063d298f00 100644
--- a/.devcontainer/cuda11.0-gcc9/devcontainer.json
+++ b/.devcontainer/cuda11.0-gcc9/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda11.0-gcc9"
 }
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
index d377c2e1ba9..52946df942c 100644
--- a/.devcontainer/cuda11.0-llvm9/devcontainer.json
+++ b/.devcontainer/cuda11.0-llvm9/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda11.0-llvm9"
 }
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
index 7795501421a..c8d8d19c4f3 100644
--- a/.devcontainer/cuda12.1-gcc10/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc10"
 }
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
index 33ce4225b41..5d31a63dfc0 100644
--- a/.devcontainer/cuda12.1-gcc11/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc11"
 }
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
index cb518a231cc..f66384fc7d8 100644
--- a/.devcontainer/cuda12.1-gcc12/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc12"
 }
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
index 4372c2333e2..b6b6f982949 100644
--- a/.devcontainer/cuda12.1-gcc7/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc7"
 }
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
index 1c2171a821b..6fffb8aff1f 100644
--- a/.devcontainer/cuda12.1-gcc8/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc8"
 }
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
index 72ecb726225..a68bc776134 100644
--- a/.devcontainer/cuda12.1-gcc9/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-gcc9"
 }
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
index bd541afd5ec..10307b85907 100644
--- a/.devcontainer/cuda12.1-llvm10/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm10"
 }
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
index 292c2138b41..5bda646de84 100644
--- a/.devcontainer/cuda12.1-llvm11/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm11"
 }
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
index 55caa622f8d..748aec1361d 100644
--- a/.devcontainer/cuda12.1-llvm12/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm12"
 }
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
index 4b7792580cd..2b4652a454d 100644
--- a/.devcontainer/cuda12.1-llvm13/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm13"
 }
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
index 15b905b1508..eb4eee24b2b 100644
--- a/.devcontainer/cuda12.1-llvm14/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm14"
 }
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
index 3a18b9a8843..8bbcfaf933a 100644
--- a/.devcontainer/cuda12.1-llvm15/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm15"
 }
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
index ae74bd6de22..d971fff7795 100644
--- a/.devcontainer/cuda12.1-llvm9/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -17,5 +17,6 @@
   "mounts": [
     "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+  ],
+  "name": "cuda12.1-llvm9"
 }
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
index dde0d9a1c04..7e5e430cfc5 100755
--- a/.devcontainer/make_devcontainers.sh
+++ b/.devcontainer/make_devcontainers.sh
@@ -29,17 +29,14 @@ for combination in $combinations; do
     compiler_version=$(echo "$combination" | jq -r '.compiler_version')
     ubuntu_version=$(echo "$combination" | jq -r '.ubuntu')
 
-    directory="cuda$cuda_version-$compiler_name$compiler_version"
-
-    mkdir -p "$directory"
-
-    devcontainer_file="$directory/devcontainer.json"
-
-    image_name="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
+    name="cuda$cuda_version-$compiler_name$compiler_version"
+    mkdir -p "$name"
+    devcontainer_file="$name/devcontainer.json"
+    image="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
 
     # Use the base_devcontainer.json as a template, plug in the CUDA, compiler names, versions, and Ubuntu version,
     # and write the output to the new devcontainer.json file
-    jq --arg image_name "$image_name" '. + {image: $image_name}' $base_devcontainer_file > "$devcontainer_file"
+    jq --arg image "$image"  --arg name "$name" '. + {image: $image, name: $name}' $base_devcontainer_file > "$devcontainer_file"
 
     echo "Created $devcontainer_file"
 done

From 781226538b36f5dacd88fcfb0d52ff5d4bd13053 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:05:04 +0000
Subject: [PATCH 305/395] Use cmake options for ignoring deprecation warnings.

---
 ci/build_thrust.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 9505a552103..803541dd266 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -46,8 +46,8 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
-      -DCMAKE_CXX_FLAGS="-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT -DCUB_IGNORE_DEPRECATED_CPP_DIALECT" \
-      -DCMAKE_CUDA_FLAGS="-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT -DCUB_IGNORE_DEPRECATED_CPP_DIALECT" \
+      -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT=ON \
+      -DCUB_IGNORE_DEPRECATED_CPP_DIALECT=ON \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \

From a5feaa4650d180014f53f0e293169456a0b9059b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:09:40 +0000
Subject: [PATCH 306/395] fromJSON on gpu archs

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 05a493b3352..4b33b226241 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -24,7 +24,7 @@ jobs:
       compiler_exe: ${{ matrix.compiler.exe }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
-      gpu_build_archs: ${{ matrix.gpu_build_archs }} 
+      gpu_build_archs: ${{ fromJSON(matrix.gpu_build_archs) }}
       cpu: ${{ matrix.cpu }}
       ubuntu: ${{ matrix.ubuntu }}
       build_script: ${{ inputs.build_script }}

From 88c78757e719d22a7cb5d39e2e78063ba3221c68 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:14:49 +0000
Subject: [PATCH 307/395] Use string for GPU archs.

---
 ci/matrix.yaml | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 63a320f0833..a9efd3cea5a 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,20 +22,20 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: ['70;80'], std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}

From 5607728428946a8f7b522f10652e5f02b99cdc05 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:17:21 +0000
Subject: [PATCH 308/395] Single quotes.

---
 ci/matrix.yaml | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index a9efd3cea5a..3e63b3c75db 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,20 +22,20 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: "70;80", std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}

From 0094a7f63a919c49d57e74ecfd639bb17bb74bbc Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:19:20 +0000
Subject: [PATCH 309/395] remove the from JSON.

---
 .github/workflows/dispatch-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 4b33b226241..3b129bcb036 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -24,7 +24,7 @@ jobs:
       compiler_exe: ${{ matrix.compiler.exe }}
       compiler_version: ${{ matrix.compiler.version }}
       std: ${{ matrix.std }}
-      gpu_build_archs: ${{ fromJSON(matrix.gpu_build_archs) }}
+      gpu_build_archs: ${{ matrix.gpu_build_archs }}
       cpu: ${{ matrix.cpu }}
       ubuntu: ${{ matrix.ubuntu }}
       build_script: ${{ inputs.build_script }}

From d405adcd2704f6be539515aa69a0197ca90fcdea Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 16:21:56 +0000
Subject: [PATCH 310/395] Update comment on matrix

---
 ci/matrix.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 3e63b3c75db..7667a18b21a 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -16,7 +16,7 @@ gpus:
 #   name: The compiler name
 #   version: The compiler version
 #   exe: The unverionsed compiler binary name
-# gpu_build_archs: The GPU architectures to build for
+# gpu_build_archs: The GPU architectures to build for using the syntax of CMAKE_CUDA_ARCHITECTURES (semi-colon separated list)
 # std: The C++ standards to build for
 #    This field is unique as it will generate an independent build/test job for each value
 

From a181ef217c410e6445948be999bf9fe7b38d2ac5 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 12:36:42 -0400
Subject: [PATCH 311/395] Remove temporary warning suppression from Thrust/CUB.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index 7bfe37a4a05..fa8bc6a74e8 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 7bfe37a4a050bee834e325190bac0fb893746e7a
+Subproject commit fa8bc6a74e8b36ada0d47991b3fdbfb3f9d32da3
diff --git a/thrust b/thrust
index 572c19cef64..77c008ab2bf 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 572c19cef64b3783c55ddbd60fc45106ed36d251
+Subproject commit 77c008ab2bf6e43766527ce26ca9879ca576eb32

From b713ce9a0eeb7b47ab0a67f386a5fe6f92c7416a Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 13:30:13 -0400
Subject: [PATCH 312/395] Add `LANGUAGES` keyword to CMake project calls.

---
 CMakeLists.txt                        | 2 +-
 cub                                   | 2 +-
 libcudacxx                            | 2 +-
 test/cmake/test_export/CMakeLists.txt | 2 +-
 thrust                                | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 529f8f69a8e..ef488320590 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,7 +8,7 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_LIST_DIR}")
   set(CCCL_TOPLEVEL_PROJECT ON)
 endif()
 
-project(CCCL NONE)
+project(CCCL LANGUAGES NONE)
 
 # Optionally include installation rules for non-top-level builds:
 option(CCCL_ENABLE_INSTALL_RULES "Enable installation of CCCL." ${CCCL_TOPLEVEL_PROJECT})
diff --git a/cub b/cub
index fa8bc6a74e8..8ae09d6b397 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit fa8bc6a74e8b36ada0d47991b3fdbfb3f9d32da3
+Subproject commit 8ae09d6b397f45a32b308345817662e08b8957de
diff --git a/libcudacxx b/libcudacxx
index f6a0035d206..d4a5476dd45 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit f6a0035d2069684ee503a12f3594db545dd8d03d
+Subproject commit d4a5476dd458fc3b9f29c4b3dfca08e9adaea05a
diff --git a/test/cmake/test_export/CMakeLists.txt b/test/cmake/test_export/CMakeLists.txt
index b10ea675430..cfa86f2e3a4 100644
--- a/test/cmake/test_export/CMakeLists.txt
+++ b/test/cmake/test_export/CMakeLists.txt
@@ -14,7 +14,7 @@
 
 
 cmake_minimum_required(VERSION 3.21)
-project(CCCLTestExport CXX)
+project(CCCLTestExport LANGUAGES CXX)
 
 include(CTest)
 enable_testing()
diff --git a/thrust b/thrust
index 77c008ab2bf..2b3694f0582 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 77c008ab2bf6e43766527ce26ca9879ca576eb32
+Subproject commit 2b3694f058204164e1f54bbabdc66287862940b7

From 3011b3c99b51e689f5a6260856a659bb76e557d0 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 13:31:58 -0400
Subject: [PATCH 313/395] Simplify some genexes and update compiler ID where
 needed.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index 8ae09d6b397..0e616626af5 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 8ae09d6b397f45a32b308345817662e08b8957de
+Subproject commit 0e616626af57691fe8d20e0c40bb93b905d35adc
diff --git a/thrust b/thrust
index 2b3694f0582..fab7dbb0b05 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 2b3694f058204164e1f54bbabdc66287862940b7
+Subproject commit fab7dbb0b05111aa70de6a5ba3e76db0fd478d58

From f778ea0eea9be37e4f5ac7ba5a3b545f5c635c8a Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 13:33:55 -0400
Subject: [PATCH 314/395] Remove unnecessary NVC++ tweaks.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index 0e616626af5..6f80a1693dc 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 0e616626af57691fe8d20e0c40bb93b905d35adc
+Subproject commit 6f80a1693dc1dffe0a13c4e1ef2d8bc703345410
diff --git a/thrust b/thrust
index fab7dbb0b05..af41b44c795 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit fab7dbb0b05111aa70de6a5ba3e76db0fd478d58
+Subproject commit af41b44c795da0f878fd6c93110a39545e04c11e

From 3a5c3240a2e0be4c3313d16432dbf6bddcd13ce1 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 14:34:38 -0400
Subject: [PATCH 315/395] Add guards against unsupported arch options.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index 6f80a1693dc..7af4b33eca7 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 6f80a1693dc1dffe0a13c4e1ef2d8bc703345410
+Subproject commit 7af4b33eca7361ca3366238e45b4f795c2f71d49
diff --git a/thrust b/thrust
index af41b44c795..4d43a76b62a 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit af41b44c795da0f878fd6c93110a39545e04c11e
+Subproject commit 4d43a76b62a2365edd50a0d6a04279b18fc93994

From 05d2594a5672bb72c54bfa7875b385201852f5ce Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Mon, 5 Jun 2023 14:41:01 -0400
Subject: [PATCH 316/395] Enable PIC for RDC targets.

---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index 7af4b33eca7..9d4a3668472 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 7af4b33eca7361ca3366238e45b4f795c2f71d49
+Subproject commit 9d4a3668472f70df4faee939c7add2587064c58c
diff --git a/thrust b/thrust
index 4d43a76b62a..b3827cb76e2 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 4d43a76b62a2365edd50a0d6a04279b18fc93994
+Subproject commit b3827cb76e2a9c5f1594d78263eace4c4e094f6c

From c98570af3a79ee79b12362bedc19a370c47d304c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 21:38:44 +0000
Subject: [PATCH 317/395] Make os in matrix more general.

---
 .github/workflows/build-and-test.yml          |  4 +--
 .github/workflows/dispatch-build-and-test.yml |  2 +-
 ci/matrix.yaml                                | 34 +++++++++----------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 56ed75b4056..f8c709bd1de 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -14,7 +14,7 @@ on:
       std: {type: string, required: true}
       gpu_build_archs: {type: string, required: true}
       cpu: {type: string, required: true}
-      ubuntu: {type: string, required: true}
+      os: {type: string, required: true}
       build_script: {type: string, required: false}
       test_script: {type: string, required: false }
 
@@ -25,7 +25,7 @@ jobs:
     runs-on: linux-${{inputs.cpu}}-cpu16
     container: 
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-ubuntu${{ inputs.ubuntu }}
+      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
     permissions:
       id-token: write
       contents: read
diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 3b129bcb036..c5f1e955fe8 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -26,7 +26,7 @@ jobs:
       std: ${{ matrix.std }}
       gpu_build_archs: ${{ matrix.gpu_build_archs }}
       cpu: ${{ matrix.cpu }}
-      ubuntu: ${{ matrix.ubuntu }}
+      os: ${{ matrix.os }}
       build_script: ${{ inputs.build_script }}
       test_script: ${{ inputs.test_script }}
 
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 7667a18b21a..6922ca8223c 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,20 +22,20 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '11.0', ubuntu: '18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', ubuntu: '22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}

From d089ac0010393e0f7a35ae82711787c1348701dc Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 21:43:07 +0000
Subject: [PATCH 318/395] Update os docs.

---
 ci/matrix.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 6922ca8223c..e949e42fd74 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -10,7 +10,7 @@ gpus:
 # Each environment below will generate a unique build/test job
 # See the "compute-matrix" job in the workflow for how this is parsed and used
 # cuda: The CUDA Toolkit version
-# ubuntu: The Ubuntu version
+# os: The operating system used
 # cpu: The CPU architecture
 # compiler: The compiler to use
 #   name: The compiler name

From 2c065008d665c20e64bc34420a632bdec6c5b11b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 21:50:01 +0000
Subject: [PATCH 319/395] Test build-and-test with windows support.

---
 .github/workflows/build-and-test.yml | 100 +++++++++++++++------------
 ci/matrix.yaml                       |   1 +
 2 files changed, 56 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index f8c709bd1de..06076c80a74 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,53 +22,63 @@ jobs:
   build:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: linux-${{inputs.cpu}}-cpu16
+    runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || ('linux-' + inputs.cpu + '-cpu16') }}
     container: 
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
+      options: ${{ (inputs.os == 'linux' && '-u root') || '' }}
+      image: ${{ (inputs.os == 'linux' && 'rapidsai/devcontainers:23.06-cpp-' + inputs.compiler + inputs.compiler_version + '-cuda' + inputs.cuda_version + '-' + inputs.os) || '' }}
     permissions:
       id-token: write
       contents: read
     steps:
-      - name: Echo git version
-        run: git --version
-      - name: Update git
-        run: | 
-          sudo add-apt-repository -y ppa:git-core/ppa
-          sudo apt-get update
-          sudo apt-get install -y git
-          git --version
-      - name: Checkout repo
-        uses: actions/checkout@v3
-        with:
-          submodules: 'recursive'
-      - name: Configure credentials and environment variables for sccache
-        uses: ./.github/actions/configure_cccl_sccache
-      - name: Run build script
-        run: | 
-            sccache -s
-            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
-            sccache -s
-  test:
-    needs: build
-    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
-    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: ubuntu-latest
-    #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
-    #container: # Fetch the container image to use for the job
-    #  options: -u root
-    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - name: Configure credentials and environment variables for sccache
-        uses: ./.github/actions/configure_cccl_sccache
-      - name: Run test script
-        run: | 
-            time ./${{ inputs.test_script }}
+      - name: Echo Hello on Windows
+        if: inputs.os == 'windows'
+        shell: powershell
+        run: echo "Hello"
+      - name: Hellon on linux
+      if: inputs.os == 'linux'
+      shell: bash
+      run: echo "Hello"
+
+#      - name: Echo git version
+#        run: git --version
+#      - name: Update git
+#        run: | 
+#          sudo add-apt-repository -y ppa:git-core/ppa
+#          sudo apt-get update
+#          sudo apt-get install -y git
+#          git --version
+#      - name: Checkout repo
+#        uses: actions/checkout@v3
+#        with:
+#          submodules: 'recursive'
+#      - name: Configure credentials and environment variables for sccache
+#        uses: ./.github/actions/configure_cccl_sccache
+#      - name: Run build script
+#        run: | 
+#            sccache -s
+#            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
+#            sccache -s
+#  test:
+#    needs: build
+#    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
+#    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
+#    runs-on: ubuntu-latest
+#    #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
+#    #container: # Fetch the container image to use for the job
+#    #  options: -u root
+#    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
+#    permissions:
+#      id-token: write
+#      contents: read
+#    steps:
+#      - name: Checkout repo
+#        uses: actions/checkout@v3
+#        with:
+#          fetch-depth: 0
+#          persist-credentials: false
+#      - name: Configure credentials and environment variables for sccache
+#        uses: ./.github/actions/configure_cccl_sccache
+#      - name: Run test script
+#        run: | 
+#            time ./${{ inputs.test_script }}
+#
\ No newline at end of file
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index e949e42fd74..243192b9c90 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -39,3 +39,4 @@ pull-request:
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'windows', cpu: 'amd64', compiler: {name: 'msvc', version: '17', exe: 'cl.exe'}, gpu_build_archs: '70;80', std: [17]}

From ef2f34eaf136bfecfee7f45c77dd4eb4a8306cd1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 21:51:05 +0000
Subject: [PATCH 320/395] Syntax

---
 .github/workflows/build-and-test.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 06076c80a74..53711cb9ce2 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -30,11 +30,11 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Echo Hello on Windows
-        if: inputs.os == 'windows'
-        shell: powershell
-        run: echo "Hello"
-      - name: Hellon on linux
+    - name: Echo Hello on Windows
+      if: inputs.os == 'windows'
+      shell: powershell
+      run: echo "Hello"
+    - name: Hellon on linux
       if: inputs.os == 'linux'
       shell: bash
       run: echo "Hello"

From 12946732418b298df47497f733c1cff6fd28ac81 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 21:53:13 +0000
Subject: [PATCH 321/395] Factor into env.

---
 .github/workflows/build-and-test.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 53711cb9ce2..6d079fcd6d9 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,10 +22,14 @@ jobs:
   build:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || ('linux-' + inputs.cpu + '-cpu16') }}
+    runs-on: ${{ env.RUNS_ON }}
+    env:
+      RUNS_ON: ${{ (inputs.os == 'windows' && 'windows-latest') || ('linux-' + inputs.cpu + '-cpu16') }}
+      CONTAINER_OPTIONS: ${{ (inputs.os == 'linux' && '-u root') || '' }}
+      CONTAINER_IMAGE: ${{ (inputs.os == 'linux' && 'rapidsai/devcontainers:23.06-cpp-' + inputs.compiler + inputs.compiler_version + '-cuda' + inputs.cuda_version + '-' + inputs.os) || '' }}
     container: 
-      options: ${{ (inputs.os == 'linux' && '-u root') || '' }}
-      image: ${{ (inputs.os == 'linux' && 'rapidsai/devcontainers:23.06-cpp-' + inputs.compiler + inputs.compiler_version + '-cuda' + inputs.cuda_version + '-' + inputs.os) || '' }}
+      options: ${{ env.CONTAINER_OPTIONS }}
+      image: ${{ env.CONTAINER_IMAGE }}
     permissions:
       id-token: write
       contents: read

From f6903358d56299145e5c6fe0a9c9556367856fa8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:04:10 +0000
Subject: [PATCH 322/395] Update to use format().

---
 .github/workflows/build-and-test.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6d079fcd6d9..ec7e2d540cb 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -22,14 +22,10 @@ jobs:
   build:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: ${{ env.RUNS_ON }}
-    env:
-      RUNS_ON: ${{ (inputs.os == 'windows' && 'windows-latest') || ('linux-' + inputs.cpu + '-cpu16') }}
-      CONTAINER_OPTIONS: ${{ (inputs.os == 'linux' && '-u root') || '' }}
-      CONTAINER_IMAGE: ${{ (inputs.os == 'linux' && 'rapidsai/devcontainers:23.06-cpp-' + inputs.compiler + inputs.compiler_version + '-cuda' + inputs.cuda_version + '-' + inputs.os) || '' }}
+    runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
     container: 
-      options: ${{ env.CONTAINER_OPTIONS }}
-      image: ${{ env.CONTAINER_IMAGE }}
+      options: ${{ (inputs.os == 'linux' && '-u root') || '' }}
+      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || '' }}
     permissions:
       id-token: write
       contents: read

From 36c83428c6d331ea087eaf846d4bb5a5d6c59bb2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:08:59 +0000
Subject: [PATCH 323/395] Make the whole container line optionally empty.

---
 .github/workflows/build-and-test.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index ec7e2d540cb..6451ed5fc6f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,9 +23,7 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    container: 
-      options: ${{ (inputs.os == 'linux' && '-u root') || '' }}
-      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || '' }}
+    container: ${{ (inputs.os == 'linux' && { options: '-u root', image: format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os) }) || toJson('') }}
     permissions:
       id-token: write
       contents: read

From 7779208d9a5f9c30d807025f46664a1a64239178 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:13:41 +0000
Subject: [PATCH 324/395] All in format().

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 6451ed5fc6f..2f9416b3b53 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,7 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    container: ${{ (inputs.os == 'linux' && { options: '-u root', image: format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os) }) || toJson('') }}
+    container: ${{ (inputs.os == 'linux' && format('{ options: '-u root', image: rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os) }) || '' }}
     permissions:
       id-token: write
       contents: read

From 5ae412e677a3fc76ae933e66f8ea3b0e7384daf8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:15:28 +0000
Subject: [PATCH 325/395] format

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2f9416b3b53..5f0bfd8af86 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,7 +23,7 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    container: ${{ (inputs.os == 'linux' && format('{ options: '-u root', image: rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os) }) || '' }}
+    container: ${{ (inputs.os == 'linux' && format('{{"options": "-u root", "image": "rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}"}}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
     permissions:
       id-token: write
       contents: read

From 116a594bf1bc1f206f18f2bd20d7bdcec068f990 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:28:32 +0000
Subject: [PATCH 326/395] Don't specify option.

---
 .github/workflows/build-and-test.yml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 5f0bfd8af86..8e6265229bc 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,19 +23,20 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    container: ${{ (inputs.os == 'linux' && format('{{"options": "-u root", "image": "rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}"}}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
+    container: 
+      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
     permissions:
       id-token: write
       contents: read
     steps:
-    - name: Echo Hello on Windows
-      if: inputs.os == 'windows'
-      shell: powershell
-      run: echo "Hello"
-    - name: Hellon on linux
-      if: inputs.os == 'linux'
-      shell: bash
-      run: echo "Hello"
+      - name: Echo Hello on Windows
+        if: inputs.os == 'windows'
+        shell: powershell
+        run: echo "Hello"
+      - name: Hellon on linux
+        if: inputs.os == 'linux'
+        shell: bash
+        run: echo "Hello"
 
 #      - name: Echo git version
 #        run: git --version

From 7881447c25e54b76dafb4eba2e63fdb87391531f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 22:59:31 +0000
Subject: [PATCH 327/395] test specifying workflow as input.

---
 .github/workflows/dispatch-build-and-test.yml | 4 +++-
 .github/workflows/pr.yml                      | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index c5f1e955fe8..879153671e1 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -6,6 +6,7 @@ on:
       per_cuda_compiler_matrix: {type: string, required: true}
       build_script: {type: string, required: false}
       test_script: {type: string, required: false}
+      workflow: {type: string, required: true}
 
 jobs:
   # Using a matrix to dispatch to the build-and-test reusable workflow for each build configuration
@@ -13,7 +14,8 @@ jobs:
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
     name: ${{matrix.cpu}}
-    uses: ./.github/workflows/build-and-test.yml
+    uses: ${{inputs.workflow}}
+    #uses: ./.github/workflows/build-and-test.yml
     strategy:
       fail-fast: false
       matrix: 
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 1f3168c636f..746fb80a080 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -37,6 +37,7 @@ jobs:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: #"./ci/test_thrust.sh"
+      workflow: ./.github/workflows/build-and-test.yml
 
   #cub:
   #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}

From 2bfaabc19a85b234e4d1021c3a0a6eb42b45d87f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Mon, 5 Jun 2023 23:00:42 +0000
Subject: [PATCH 328/395] quotes

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 746fb80a080..aa7310a0da6 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -37,7 +37,7 @@ jobs:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: #"./ci/test_thrust.sh"
-      workflow: ./.github/workflows/build-and-test.yml
+      workflow: "./.github/workflows/build-and-test.yml"
 
   #cub:
   #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}

From 27c7e209350ee549f51fea16680e4c86cb408d58 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:17:30 +0000
Subject: [PATCH 329/395] Can't dynamically specify "uses:"

---
 .github/workflows/dispatch-build-and-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 879153671e1..8b21c295aa7 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -14,8 +14,7 @@ jobs:
   # the build step for CUDA 12.1 + gcc 9.3 can run at the same time as the test step for CUDA 11.0 + clang 11.
   build_and_test:
     name: ${{matrix.cpu}}
-    uses: ${{inputs.workflow}}
-    #uses: ./.github/workflows/build-and-test.yml
+    uses: ./.github/workflows/build-and-test.yml
     strategy:
       fail-fast: false
       matrix: 

From ffb2d57050fd26a2beedf15c992fe32979d9e544 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:24:44 +0000
Subject: [PATCH 330/395] Set image name in env variable.

---
 .github/workflows/build-and-test.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8e6265229bc..25ded303459 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,8 +23,10 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
+    env:
+      CONTAINER_IMAGE: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
     container: 
-      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
+      image: ${{ env.CONTAINER_IMAGE }}
     permissions:
       id-token: write
       contents: read

From 86640fdf6a816e923688e9dfc31c318486c1f955 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:28:42 +0000
Subject: [PATCH 331/395] Try specifying options even if image is empty.

---
 .github/workflows/build-and-test.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 25ded303459..25a5c5292b8 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,10 +23,9 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    env:
-      CONTAINER_IMAGE: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
     container: 
-      image: ${{ env.CONTAINER_IMAGE }}
+      options: -u root
+      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
     permissions:
       id-token: write
       contents: read

From 40ae5f26e4de0b144215989ea992998cb7bfa219 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:52:07 +0000
Subject: [PATCH 332/395] Remove stale "workflow:" input

---
 .github/workflows/dispatch-build-and-test.yml | 1 -
 .github/workflows/pr.yml                      | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index 8b21c295aa7..c5f1e955fe8 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -6,7 +6,6 @@ on:
       per_cuda_compiler_matrix: {type: string, required: true}
       build_script: {type: string, required: false}
       test_script: {type: string, required: false}
-      workflow: {type: string, required: true}
 
 jobs:
   # Using a matrix to dispatch to the build-and-test reusable workflow for each build configuration
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index aa7310a0da6..36e34a18d4c 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -24,7 +24,7 @@ jobs:
       matrix_file: "./ci/matrix.yaml"
       matrix_type: "pull-request"
 
-  thrust:
+  thrust-linux:
     name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml

From dbeebb4e105650af5dee8f2b4d500104c56c68c6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:56:03 +0000
Subject: [PATCH 333/395] Actually remove the workflow: input.

---
 .github/workflows/pr.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 36e34a18d4c..36b45da342a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -37,7 +37,6 @@ jobs:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: #"./ci/test_thrust.sh"
-      workflow: "./.github/workflows/build-and-test.yml"
 
   #cub:
   #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}

From 4d869458cb536046c8d39b9d32a28bf2f7d50a87 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 19:56:40 +0000
Subject: [PATCH 334/395] Don't change the name of the Thrust job.

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 36b45da342a..1f3168c636f 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -24,7 +24,7 @@ jobs:
       matrix_file: "./ci/matrix.yaml"
       matrix_type: "pull-request"
 
-  thrust-linux:
+  thrust:
     name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
     needs: compute-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml

From af74d4ef0f144aae7adeb7d480feec6076a62f4e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 20:36:01 +0000
Subject: [PATCH 335/395] Enable thrust test job.

---
 .github/workflows/build-and-test.yml | 100 +++++++++++++--------------
 .github/workflows/pr.yml             |   2 +-
 ci/build_thrust.sh                   |   1 -
 ci/test_thrust.sh                    |  25 ++-----
 4 files changed, 55 insertions(+), 73 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 25a5c5292b8..715bbad2a2b 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -30,55 +30,51 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Echo Hello on Windows
-        if: inputs.os == 'windows'
-        shell: powershell
-        run: echo "Hello"
-      - name: Hellon on linux
-        if: inputs.os == 'linux'
-        shell: bash
-        run: echo "Hello"
-
-#      - name: Echo git version
-#        run: git --version
-#      - name: Update git
-#        run: | 
-#          sudo add-apt-repository -y ppa:git-core/ppa
-#          sudo apt-get update
-#          sudo apt-get install -y git
-#          git --version
-#      - name: Checkout repo
-#        uses: actions/checkout@v3
-#        with:
-#          submodules: 'recursive'
-#      - name: Configure credentials and environment variables for sccache
-#        uses: ./.github/actions/configure_cccl_sccache
-#      - name: Run build script
-#        run: | 
-#            sccache -s
-#            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
-#            sccache -s
-#  test:
-#    needs: build
-#    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
-#    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-#    runs-on: ubuntu-latest
-#    #runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
-#    #container: # Fetch the container image to use for the job
-#    #  options: -u root
-#    #  image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-#    permissions:
-#      id-token: write
-#      contents: read
-#    steps:
-#      - name: Checkout repo
-#        uses: actions/checkout@v3
-#        with:
-#          fetch-depth: 0
-#          persist-credentials: false
-#      - name: Configure credentials and environment variables for sccache
-#        uses: ./.github/actions/configure_cccl_sccache
-#      - name: Run test script
-#        run: | 
-#            time ./${{ inputs.test_script }}
-#
\ No newline at end of file
+      - name: Echo git version
+        run: git --version
+      - name: Update git
+        run: | 
+          sudo add-apt-repository -y ppa:git-core/ppa
+          sudo apt-get update
+          sudo apt-get install -y git
+          git --version
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+      - name: Configure credentials and environment variables for sccache
+        uses: ./.github/actions/configure_cccl_sccache
+      - name: Run build script
+        run: | 
+            sccache -s
+            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
+            sccache -s
+  test:
+    needs: build
+    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
+    name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
+    runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
+    container: 
+      options: -u root
+      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Update git
+        run: | 
+          sudo add-apt-repository -y ppa:git-core/ppa
+          sudo apt-get update
+          sudo apt-get install -y git
+          git --version
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+      - name: Configure credentials and environment variables for sccache
+        uses: ./.github/actions/configure_cccl_sccache
+      - name: Run test script
+        run: | 
+            sccache -s
+            time ./${{ inputs.test_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
+            sccache -s
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 1f3168c636f..dfe923eab1a 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -36,7 +36,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
-      test_script: #"./ci/test_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
 
   #cub:
   #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 803541dd266..68e9ca9810e 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -5,7 +5,6 @@ set -xeuo pipefail
 # Ensure the script is being executed in its containing directory
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
-
 # Check if the correct number of arguments has been provided
 if [ "$#" -ne 3 ]; then
     echo "Usage: ./build_thrust.sh <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
diff --git a/ci/test_thrust.sh b/ci/test_thrust.sh
index 1433ad049df..d38cf5f706c 100755
--- a/ci/test_thrust.sh
+++ b/ci/test_thrust.sh
@@ -1,26 +1,13 @@
 #!/bin/bash
 
-echo "test thrust"
+set -xeuo pipefail
 
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
+./build_thrust.sh "$@"
 
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+ctest --test-dir ../build --output-on-failure
 
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
+echo "Thrust test complete"
 
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure

From fbe793f16c8d8060858f7baf03f31d0e63cc6373 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 20:46:23 +0000
Subject: [PATCH 336/395] Don't update git.

---
 .github/workflows/build-and-test.yml | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 715bbad2a2b..d3adcf10f48 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -25,19 +25,13 @@ jobs:
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
     container: 
       options: -u root
-      image: ${{ (inputs.os == 'linux' && format('rapidsai/devcontainers:23.06-cpp-{0}{1}-cuda{2}-{3}', inputs.compiler, inputs.compiler_version, inputs.cuda_version, inputs.os)) || null }}
+      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
     permissions:
       id-token: write
       contents: read
     steps:
       - name: Echo git version
         run: git --version
-      - name: Update git
-        run: | 
-          sudo add-apt-repository -y ppa:git-core/ppa
-          sudo apt-get update
-          sudo apt-get install -y git
-          git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
@@ -61,12 +55,6 @@ jobs:
       id-token: write
       contents: read
     steps:
-      - name: Update git
-        run: | 
-          sudo add-apt-repository -y ppa:git-core/ppa
-          sudo apt-get update
-          sudo apt-get install -y git
-          git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:

From f20375e118f5a65fba50dc923829f6fa8152c41f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 20:48:30 +0000
Subject: [PATCH 337/395] Remove windows from the matrix for now.

---
 ci/matrix.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 243192b9c90..e949e42fd74 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -39,4 +39,3 @@ pull-request:
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'windows', cpu: 'amd64', compiler: {name: 'msvc', version: '17', exe: 'cl.exe'}, gpu_build_archs: '70;80', std: [17]}

From 4fa557315c6c966c30906f7071d5fc8ab8e2efe6 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 20:50:46 +0000
Subject: [PATCH 338/395] Update git version

---
 .github/workflows/build-and-test.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d3adcf10f48..8b08d54b903 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -32,6 +32,13 @@ jobs:
     steps:
       - name: Echo git version
         run: git --version
+      - name: Update git version
+        run: |
+          sudo apt-get install python-software-properties software-properties-common
+          sudo add-apt-repository ppa:git-core/ppa -y
+          sudo apt-get update
+          sudo apt-get install git -y
+          git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:
@@ -55,6 +62,13 @@ jobs:
       id-token: write
       contents: read
     steps:
+      - name: Update git version
+        run: |
+          sudo apt-get install python-software-properties software-properties-common
+          sudo add-apt-repository ppa:git-core/ppa -y
+          sudo apt-get update
+          sudo apt-get install git -y
+          git --version
       - name: Checkout repo
         uses: actions/checkout@v3
         with:

From 16e9f417980bb210bf4aaf59c2a5f7b400d204b5 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 20:53:45 +0000
Subject: [PATCH 339/395] Get rid of outdated apt package.

---
 .github/workflows/build-and-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 8b08d54b903..737f8aec1c1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -34,7 +34,7 @@ jobs:
         run: git --version
       - name: Update git version
         run: |
-          sudo apt-get install python-software-properties software-properties-common
+          sudo apt-get install software-properties-common
           sudo add-apt-repository ppa:git-core/ppa -y
           sudo apt-get update
           sudo apt-get install git -y
@@ -64,7 +64,7 @@ jobs:
     steps:
       - name: Update git version
         run: |
-          sudo apt-get install python-software-properties software-properties-common
+          sudo apt-get install software-properties-common
           sudo add-apt-repository ppa:git-core/ppa -y
           sudo apt-get update
           sudo apt-get install git -y

From 2d472fd39f9b0f650483837daf09ed314ebe9207 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 6 Jun 2023 21:13:02 +0000
Subject: [PATCH 340/395] Add NVIDIA_VISIBLE_DEVICES env to test job container.

---
 .github/workflows/build-and-test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 737f8aec1c1..a5017a0fc96 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -58,10 +58,13 @@ jobs:
     container: 
       options: -u root
       image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
     permissions:
       id-token: write
       contents: read
     steps:
+    # TODO: We can remove this when we aren't using submodules anymore
       - name: Update git version
         run: |
           sudo apt-get install software-properties-common

From 8f51aaad2f4852171675f2ef3ad8cf6539d5acc5 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Tue, 6 Jun 2023 17:15:33 -0400
Subject: [PATCH 341/395] Update more CUDA compiler id checks.

---
 cub        | 2 +-
 libcudacxx | 2 +-
 thrust     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cub b/cub
index 9d4a3668472..1d94385d0fb 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 9d4a3668472f70df4faee939c7add2587064c58c
+Subproject commit 1d94385d0fbecf60c29414903e761d6b14758560
diff --git a/libcudacxx b/libcudacxx
index d4a5476dd45..15ff8f2e857 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit d4a5476dd458fc3b9f29c4b3dfca08e9adaea05a
+Subproject commit 15ff8f2e857301f769fa42bec89401d5e4edabac
diff --git a/thrust b/thrust
index b3827cb76e2..0d97ac1ff3e 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit b3827cb76e2a9c5f1594d78263eace4c4e094f6c
+Subproject commit 0d97ac1ff3e512e5c8e1d42595984000415f44a9

From 4cd24c521250a6de06027bbd7a81748f97e0aed3 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Tue, 6 Jun 2023 18:55:35 -0400
Subject: [PATCH 342/395] Move dev build toggle to the cccl project level.

This allows the install rules to be generated for all projects even
when the dev builds are disabled.
---
 CMakeLists.txt | 10 +++++++---
 cub            |  2 +-
 libcudacxx     |  2 +-
 thrust         |  2 +-
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ef488320590..4be3ba2e6d7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -34,17 +34,21 @@ include(CTest)
 enable_testing()
 
 if (CCCL_ENABLE_LIBCUDACXX)
-  add_subdirectory(libcudacxx)
+  set(LIBCUDACXX_TOPLEVEL_PROJECT ON)
 endif()
 
 if (CCCL_ENABLE_CUB)
-  add_subdirectory(cub)
+  set(CUB_TOPLEVEL_PROJECT ON)
 endif()
 
 if (CCCL_ENABLE_THRUST)
-  add_subdirectory(thrust)
+  set(THRUST_TOPLEVEL_PROJECT ON)
 endif()
 
+add_subdirectory(libcudacxx)
+add_subdirectory(cub)
+add_subdirectory(thrust)
+
 if (CCCL_ENABLE_TESTING)
   add_subdirectory(test)
 endif()
diff --git a/cub b/cub
index 1d94385d0fb..bfe591eb7ef 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 1d94385d0fbecf60c29414903e761d6b14758560
+Subproject commit bfe591eb7eff22e6d26a32190667bb7578c6171c
diff --git a/libcudacxx b/libcudacxx
index 15ff8f2e857..5d377200129 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit 15ff8f2e857301f769fa42bec89401d5e4edabac
+Subproject commit 5d3772001297f62380e3bc270044b9be5afe5165
diff --git a/thrust b/thrust
index 0d97ac1ff3e..21e8f3272db 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 0d97ac1ff3e512e5c8e1d42595984000415f44a9
+Subproject commit 21e8f3272db26d41d1e20976a09764810e04e129

From d7442ea3dba6a2c9a0410b1b1ff559df89499951 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 00:58:04 +0000
Subject: [PATCH 343/395] Turn on install rules for everything.

---
 ci/build_thrust.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index 68e9ca9810e..f859d63edca 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -47,6 +47,9 @@ cmake -S .. -B ../build \
       -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
       -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT=ON \
       -DCUB_IGNORE_DEPRECATED_CPP_DIALECT=ON \
+      -Dlibcudacxx_ENABLE_INSTALL_RULES=ON \
+      -DCUB_ENABLE_INSTALL_RULES=ON \
+      -DTHRUST_ENABLE_INSTALL_RULES=ON \
       -G Ninja
       # TODO: Add this back after Thrust removes the check against it.
       #-DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \

From dfc22b37cfc4eb8e1092dbb509d99076a73bb724 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 13:24:46 +0000
Subject: [PATCH 344/395] Factor common logic out of build_thrust.sh

---
 .github/workflows/build-and-test.yml |  1 +
 ci/build_common.sh                   | 33 ++++++++++++++++++++++++++++
 ci/build_thrust.sh                   | 32 +--------------------------
 3 files changed, 35 insertions(+), 31 deletions(-)
 create mode 100644 ci/build_common.sh

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a5017a0fc96..fb38cb60d27 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -32,6 +32,7 @@ jobs:
     steps:
       - name: Echo git version
         run: git --version
+      #TODO: We can remove this when we aren't using submodules anymore
       - name: Update git version
         run: |
           sudo apt-get install software-properties-common
diff --git a/ci/build_common.sh b/ci/build_common.sh
new file mode 100644
index 00000000000..03fb5901892
--- /dev/null
+++ b/ci/build_common.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+# Check if the correct number of arguments has been provided
+if [ "$#" -ne 3 ]; then
+    echo "Usage: $0 <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
+    echo "The PARALLEL_LEVEL environment variable controls the amount of build parallelism. Default is the number of cores."
+    echo "Example: PARALLEL_LEVEL=8 $0 g++-8 14 \"70\" "
+    echo "Example: $0 clang++-8 17 \"70;75;80-virtual\" "
+    exit 1
+fi
+
+# Assign command line arguments to variables
+readonly HOST_COMPILER=$(which $1)
+readonly CXX_STANDARD=$2
+
+# Replace spaces, commas and semicolons with semicolons for CMake list
+readonly GPU_ARCHS=$(echo $3 | tr ' ,' ';')
+
+readonly PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
+
+echo "========================================"
+echo "Begin build"
+echo "pwd=$(pwd)"
+echo "HOST_COMPILER=$HOST_COMPILER"
+echo "CXX_STANDARD=$CXX_STANDARD"
+echo "GPU_ARCHS=$GPU_ARCHS"
+echo "PARALLEL_LEVEL=$PARALLEL_LEVEL"
+echo "========================================"
\ No newline at end of file
diff --git a/ci/build_thrust.sh b/ci/build_thrust.sh
index f859d63edca..72e2c291d88 100755
--- a/ci/build_thrust.sh
+++ b/ci/build_thrust.sh
@@ -1,36 +1,6 @@
 #!/bin/bash
 
-set -xeuo pipefail
-
-# Ensure the script is being executed in its containing directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
-
-# Check if the correct number of arguments has been provided
-if [ "$#" -ne 3 ]; then
-    echo "Usage: ./build_thrust.sh <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
-    echo "The PARALLEL_LEVEL environment variable controls the amount of build parallelism. Default is the number of cores."
-    echo "Example: PARALLEL_LEVEL=8 ./build_thrust.sh g++-8 14 \"70\" "
-    echo "Example: ./build_thrust.sh clang++-8 17 \"70;75;80\" "
-    exit 1
-fi
-
-# Assign command line arguments to variables
-HOST_COMPILER=$(which $1)
-CXX_STANDARD=$2
-
-# Replace spaces, commas and semicolons with semicolons for CMake list
-GPU_ARCHS=$(echo $3 | tr ' ,' ';')
-
-PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
-
-echo "========================================"
-echo "Begin Thrust build"
-echo "pwd=$(pwd)"
-echo "HOST_COMPILER=$HOST_COMPILER"
-echo "CXX_STANDARD=$CXX_STANDARD"
-echo "GPU_ARCHS=$GPU_ARCHS"
-echo "PARALLEL_LEVEL=$PARALLEL_LEVEL"
-echo "========================================"
+source "$(dirname "$0")/build_common.sh"
 
 cmake -S .. -B ../build \
       -DCCCL_ENABLE_THRUST=ON \

From 9e792fdc09591c7fc8a17e1089b675e2c3e58833 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 13:37:53 +0000
Subject: [PATCH 345/395] Add CUB build/test and enable in matrix.

---
 .github/workflows/pr.yml | 26 +++++++++++++-------------
 ci/build_cub.sh          | 29 ++++++-----------------------
 ci/test_cub.sh           | 26 ++++++--------------------
 3 files changed, 25 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index dfe923eab1a..028065bd58d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -38,19 +38,19 @@ jobs:
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 
-  #cub:
-  #  name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-  #  needs: compute-matrix
-  #  uses: ./.github/workflows/dispatch-build-and-test.yml
-  #  strategy:
-  #    fail-fast: false
-  #    matrix: 
-  #      cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-  #      compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-  #  with:
-  #    per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-  #    build_script: #"./ci/build_cub.sh"
-  #    test_script: #"./ci/test_cub.sh"
+  cub:
+    name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_script: "./ci/build_cub.sh"
+      test_script: "./ci/test_cub.sh"
   
   #libcudacxx:
   #  name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
diff --git a/ci/build_cub.sh b/ci/build_cub.sh
index 1836814c613..7a3cf15594b 100755
--- a/ci/build_cub.sh
+++ b/ci/build_cub.sh
@@ -1,28 +1,6 @@
 #!/bin/bash
 
-# Ensure the script is being executed in its containing directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
-
-echo "Begin CUB build"
-pwd
-
-# Check if the correct number of arguments has been provided
-if [ "$#" -ne 3 ]; then
-    echo "Usage: ./build_cub.sh <HOST_COMPILER> <CXX_STANDARD> <GPU_ARCHS>"
-    echo "The PARALLEL_LEVEL environment variable controls the amount of build parallelism. Default is the number of cores."
-    echo "Example: PARALLEL_LEVEL=8 ./build_cub.sh g++-8 14 \"70\" "
-    echo "Example: ./build_cub.sh clang++-8 17 \"70;75;80\" "
-    exit 1
-fi
-
-# Assign command line arguments to variables
-HOST_COMPILER=$1
-CXX_STANDARD=$2
-
-# Replace spaces, commas and semicolons with semicolons for CMake list
-GPU_ARCHS=$(echo $3 | tr ' ,' ';')
-
-PARALLEL_LEVEL=${PARALLEL_LEVEL:=$(nproc)}
+source "$(dirname "$0")/build_common.sh"
 
 cmake -S .. -B ../build \
     -DCCCL_ENABLE_THRUST=OFF \
@@ -36,6 +14,11 @@ cmake -S .. -B ../build \
     -DCUB_ENABLE_DIALECT_CPP14=$(if [[ $CXX_STANDARD -ne 14 ]]; then echo "OFF"; else echo "ON"; fi) \
     -DCUB_ENABLE_DIALECT_CPP17=$(if [[ $CXX_STANDARD -ne 17 ]]; then echo "OFF"; else echo "ON"; fi) \
     -DCUB_ENABLE_DIALECT_CPP20=$(if [[ $CXX_STANDARD -ne 20 ]]; then echo "OFF"; else echo "ON"; fi) \
+    -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT=ON \
+    -DCUB_IGNORE_DEPRECATED_CPP_DIALECT=ON \
+    -Dlibcudacxx_ENABLE_INSTALL_RULES=ON \
+    -DCUB_ENABLE_INSTALL_RULES=ON \
+    -DTHRUST_ENABLE_INSTALL_RULES=ON \
     -G Ninja
 
 # Build the tests
diff --git a/ci/test_cub.sh b/ci/test_cub.sh
index cb2f15b739e..20c82b8af24 100755
--- a/ci/test_cub.sh
+++ b/ci/test_cub.sh
@@ -1,26 +1,12 @@
 #!/bin/bash
 
-echo "test cub"
+set -xeuo pipefail
 
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
+./build_cub.sh "$@"
 
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
+ctest --test-dir ../build --output-on-failure
 
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure
+echo "CUB test complete"
\ No newline at end of file

From 8bef2c646c0af1605bb89db9d92e8721c3fbd228 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Wed, 7 Jun 2023 10:37:21 -0400
Subject: [PATCH 346/395] Remove install tests from Thrust/CUB.

These are now handled at the CCCL level, using newer and more
robust techniques.
---
 cub    | 2 +-
 thrust | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cub b/cub
index bfe591eb7ef..43d0317e901 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit bfe591eb7eff22e6d26a32190667bb7578c6171c
+Subproject commit 43d0317e901a8e75a6708b12d9f5b14cf79ffe20
diff --git a/thrust b/thrust
index 21e8f3272db..02b23b08975 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 21e8f3272db26d41d1e20976a09764810e04e129
+Subproject commit 02b23b08975a73e75519b9ca14b45846dc0ecf41

From cbca5f5f24a6a341ff18693d3c3bd4fb77754faf Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Wed, 7 Jun 2023 11:24:51 -0400
Subject: [PATCH 347/395] Rebase submodules on upstream main branches.

---
 cub        | 2 +-
 libcudacxx | 2 +-
 thrust     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cub b/cub
index 43d0317e901..1fdce39d9ce 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 43d0317e901a8e75a6708b12d9f5b14cf79ffe20
+Subproject commit 1fdce39d9ce97a4344d2459afa0a00fa202e580c
diff --git a/libcudacxx b/libcudacxx
index 5d377200129..7faca047a42 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit 5d3772001297f62380e3bc270044b9be5afe5165
+Subproject commit 7faca047a421ec12844d446f5e297061f483bfb7
diff --git a/thrust b/thrust
index 02b23b08975..d5ccadeda97 160000
--- a/thrust
+++ b/thrust
@@ -1 +1 @@
-Subproject commit 02b23b08975a73e75519b9ca14b45846dc0ecf41
+Subproject commit d5ccadeda9716c9f74cbc2a0fb1cfb230b7f3dc4

From 9dbf49873d998b2d19e9a0097fd36d46267560bb Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 15:29:42 +0000
Subject: [PATCH 348/395] only build for sm70 for now

---
 ci/matrix.yaml | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index e949e42fd74..03f7321f08d 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,20 +22,20 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70;80', std: [11, 14, 17, 20]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}

From 6f38850224e7903cc3bb3b0f35fe78e111649dc1 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 15:31:14 +0000
Subject: [PATCH 349/395] Add libcu++ test script and enable libcu++ test jobs.

---
 .github/workflows/pr.yml | 25 ++++++++++++------------
 ci/build_libcudacxx.sh   | 26 -------------------------
 ci/test_libcudacxx.sh    | 41 ++++++++++++++++++----------------------
 3 files changed, 30 insertions(+), 62 deletions(-)
 delete mode 100755 ci/build_libcudacxx.sh

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 028065bd58d..6ba945f9c43 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -52,16 +52,15 @@ jobs:
       build_script: "./ci/build_cub.sh"
       test_script: "./ci/test_cub.sh"
   
-  #libcudacxx:
-  #  name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-  #  needs: compute-matrix
-  #  uses: ./.github/workflows/dispatch-build-and-test.yml
-  #  strategy:
-  #    fail-fast: false
-  #    matrix: 
-  #      cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-  #      compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
-  #  with:
-  #    per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-  #    build_script: #"./ci/build_libcudacxx.sh"
-  #    test_script: "./ci/test_libcudacxx.sh" 
+  libcudacxx:
+    name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
+    needs: compute-matrix
+    uses: ./.github/workflows/dispatch-build-and-test.yml
+    strategy:
+      fail-fast: false
+      matrix: 
+        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+    with:
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      test_script: "./ci/test_libcudacxx.sh" 
diff --git a/ci/build_libcudacxx.sh b/ci/build_libcudacxx.sh
deleted file mode 100755
index 106049915f4..00000000000
--- a/ci/build_libcudacxx.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-echo "build libcudacxx"
-
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
-
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
-
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
-
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure
diff --git a/ci/test_libcudacxx.sh b/ci/test_libcudacxx.sh
index c26fe94c552..989f72a9ec3 100755
--- a/ci/test_libcudacxx.sh
+++ b/ci/test_libcudacxx.sh
@@ -1,26 +1,21 @@
 #!/bin/bash
 
-echo "test libcudacxx"
+source "$(dirname "$0")/build_common.sh"
+
+cmake -S .. -B ../build \
+    -DCCCL_ENABLE_THRUST=OFF \
+    -DCCCL_ENABLE_LIBCUDACXX=ON \
+    -DCCCL_ENABLE_CUB=OFF \
+    -DCCCL_ENABLE_TESTING=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_CXX_COMPILER=${HOST_COMPILER} \
+    -DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS} \
+    -DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \
+    -DLIBCUDACXX_ENABLE_LIBCUDACXX_TESTS=ON \
+    -Dlibcudacxx_ENABLE_INSTALL_RULES=ON \
+    -DCUB_ENABLE_INSTALL_RULES=ON \
+    -DTHRUST_ENABLE_INSTALL_RULES=ON \
+    -G Ninja
+
+LIBCUDACXX_SITE_CONFIG="../build/libcudacxx/test/lit.site.cfg" lit -v --no-progress-bar -Dcompute_archs=${GPU_ARCHS} -Dstd="c++${CXX_STANDARD}" ../libcudacxx/.upstream-tests/test
 
-# Require a compiler to be specified
-#if [ $# -ne 1 ]; then
-#    echo "Usage: $0 <CXX_COMPILER>"
-#    exit 1
-#fi
-
-# First argument is the CXX compiler
-#CXX_COMPILER=$1
-
-# Clone Thrust repository 
-#git clone --recursive --depth=1 https://github.com/NVIDIA/thrust.git
-
-# Configure Thrust
-#cmake -S thrust -B thrust/build -GNinja -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON -DTHRUST_ENABLE_COMPUTE_70=ON #-DCMAKE_CXX_COMPILER=${CXX_COMPILER}
-
-
-# Build Thrust tests
-#cmake --build thrust/build
-
-
-# Run Thrust tests
-#ctest --output-on-failure

From 37987698d84d2025a96b8f1cc0813c1932c18838 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Wed, 7 Jun 2023 13:49:49 -0400
Subject: [PATCH 350/395] Update how RDC flags are applied to Catch2 utilities.

---
 cub | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cub b/cub
index 1fdce39d9ce..ab1c7562bce 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit 1fdce39d9ce97a4344d2459afa0a00fa202e580c
+Subproject commit ab1c7562bceb7c8b2123556c4b077abe12e728d0

From 39dba744cf84422656d0dfe8ecd7cbd3a262e1fe Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 22:24:52 +0000
Subject: [PATCH 351/395] Add a step to calculate sccache hit rate.

---
 .github/workflows/build-and-test.yml | 14 ++++++++++----
 ci/build_common.sh                   |  0
 ci/sccache_hit_rate.sh               | 27 +++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 ci/build_common.sh
 create mode 100755 ci/sccache_hit_rate.sh

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index fb38cb60d27..3ccad786cf1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -48,9 +48,12 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run build script
         run: | 
-            sccache -s
+            sccache -s > sccache_before.txt
             time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
-            sccache -s
+            sccache -s > sccache_after.txt
+      - name: Calculate sccache hit rate
+        run: |
+          ./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
@@ -81,6 +84,9 @@ jobs:
         uses: ./.github/actions/configure_cccl_sccache
       - name: Run test script
         run: | 
-            sccache -s
+            sccache -s > sccache_before.txt
             time ./${{ inputs.test_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
-            sccache -s
+            sccache -s > sccache_after.txt
+      - name: Calculate sccache hit rate
+        run: |
+          ./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt
diff --git a/ci/build_common.sh b/ci/build_common.sh
old mode 100644
new mode 100755
diff --git a/ci/sccache_hit_rate.sh b/ci/sccache_hit_rate.sh
new file mode 100755
index 00000000000..37e46e69ed1
--- /dev/null
+++ b/ci/sccache_hit_rate.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+# Ensure two arguments are provided
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <before-file> <after-file>"
+  exit 1
+fi
+
+# Extract compile requests and cache hits from the before and after files
+requests_before=$(awk '/^Compile requests[[:space:]]+[[:digit:]]+/ {print $3}' $1)
+hits_before=$(awk '/^Cache hits[[:space:]]+[[:digit:]]+/ {print $3}' $1)
+requests_after=$(awk '/^Compile requests[[:space:]]+[[:digit:]]+/ {print $3}' $2)
+hits_after=$(awk '/^Cache hits[[:space:]]+[[:digit:]]+/ {print $3}' $2)
+
+# Calculate the differences to find out how many new requests and hits
+requests_diff=$((requests_after - requests_before))
+hits_diff=$((hits_after - hits_before))
+
+# Calculate and print the hit rate
+if [ $requests_diff -eq 0 ]; then
+    echo "No new compile requests, hit rate is not applicable"
+else
+    hit_rate=$(awk -v hits=$hits_diff -v requests=$requests_diff 'BEGIN {printf "%.2f", hits/requests * 100}')
+    echo "sccache hit rate: $hit_rate%"
+fi

From 71578a6555209988c54fba725461c47a45441285 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 22:33:04 +0000
Subject: [PATCH 352/395] Update sccache script to print file contents.

---
 ci/sccache_hit_rate.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ci/sccache_hit_rate.sh b/ci/sccache_hit_rate.sh
index 37e46e69ed1..2b6f50bb67e 100755
--- a/ci/sccache_hit_rate.sh
+++ b/ci/sccache_hit_rate.sh
@@ -8,6 +8,16 @@ if [ $# -ne 2 ]; then
   exit 1
 fi
 
+# Print the contents of the before file
+echo "=== Contents of $1 ==="
+cat $1
+echo "=== End of $1 ==="
+
+# Print the contents of the after file
+echo "=== Contents of $2 ==="
+cat $2
+echo "=== End of $2 ==="
+
 # Extract compile requests and cache hits from the before and after files
 requests_before=$(awk '/^Compile requests[[:space:]]+[[:digit:]]+/ {print $3}' $1)
 hits_before=$(awk '/^Cache hits[[:space:]]+[[:digit:]]+/ {print $3}' $1)

From 355611da16c11defcd2b386a33c6eb6d7f92293c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 22:52:51 +0000
Subject: [PATCH 353/395] Add sccache to job summary.

---
 .github/workflows/build-and-test.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 3ccad786cf1..ccc80491ba5 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -53,7 +53,8 @@ jobs:
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate
         run: |
-          ./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt
+          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt)
+          echo "The sccache hit rate was $hit_rate" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
@@ -89,4 +90,5 @@ jobs:
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate
         run: |
-          ./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt
+          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt)
+          echo "The sccache hit rate was $hit_rate" >> $GITHUB_STEP_SUMMARY

From 4c1bfb5cc362fcd8508be978e7376c3ff40c8aa9 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 23:10:35 +0000
Subject: [PATCH 354/395] Fix writing the hit rate to job summary.

---
 .github/workflows/build-and-test.yml | 8 ++++----
 ci/sccache_hit_rate.sh               | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index ccc80491ba5..b7767d2f85f 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -53,8 +53,8 @@ jobs:
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate
         run: |
-          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt)
-          echo "The sccache hit rate was $hit_rate" >> $GITHUB_STEP_SUMMARY
+          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt  2>&1 >/dev/null)
+          echo "sccache hit rate: $hit_rate" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
@@ -90,5 +90,5 @@ jobs:
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate
         run: |
-          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt)
-          echo "The sccache hit rate was $hit_rate" >> $GITHUB_STEP_SUMMARY
+          hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt  2>&1 >/dev/null)
+          echo "sccache hit rate: $hit_rate" >> $GITHUB_STEP_SUMMARY
diff --git a/ci/sccache_hit_rate.sh b/ci/sccache_hit_rate.sh
index 2b6f50bb67e..4c99ff880f7 100755
--- a/ci/sccache_hit_rate.sh
+++ b/ci/sccache_hit_rate.sh
@@ -34,4 +34,6 @@ if [ $requests_diff -eq 0 ]; then
 else
     hit_rate=$(awk -v hits=$hits_diff -v requests=$requests_diff 'BEGIN {printf "%.2f", hits/requests * 100}')
     echo "sccache hit rate: $hit_rate%"
+    # Write just the hit rate out to stderr to make it easier to get this value later
+    echo "$hit_rate" >&2
 fi

From 0ed4be5fb4e6522aed30d233d3525494e11e1df4 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 23:23:26 +0000
Subject: [PATCH 355/395] Remove set -x to only write hit_rate to stderr

---
 ci/sccache_hit_rate.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/sccache_hit_rate.sh b/ci/sccache_hit_rate.sh
index 4c99ff880f7..5333e84fae1 100755
--- a/ci/sccache_hit_rate.sh
+++ b/ci/sccache_hit_rate.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -xeuo pipefail
+set -euo pipefail
 
 # Ensure two arguments are provided
 if [ $# -ne 2 ]; then

From 2c48d274a8fcd65296d25b2194dd90f83cd6f3fb Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 23:46:32 +0000
Subject: [PATCH 356/395] Add final "CI" job that can be used as status check.

---
 .github/workflows/pr.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 6ba945f9c43..3b7fd74c0ad 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -64,3 +64,15 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       test_script: "./ci/test_libcudacxx.sh" 
+
+  # This job is the final job that runs after all other jobs and is used for branch protection status checks.
+  # See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks
+  ci:
+    runs-on: ubuntu-latest
+    name: CI
+    needs:
+      - libcudacxx
+      - cub
+      - thrust
+    steps:
+      - run: echo "CI success"

From a4406f429cc55af4a1556cb7e26695282819f4a8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 23:53:16 +0000
Subject: [PATCH 357/395] Delete old workflow files.

---
 .github/workflows/old/build_and_test.yml |  74 --------------
 .github/workflows/old/per_compiler.yml   |  35 -------
 .github/workflows/old/pr.yml             | 124 -----------------------
 3 files changed, 233 deletions(-)
 delete mode 100644 .github/workflows/old/build_and_test.yml
 delete mode 100644 .github/workflows/old/per_compiler.yml
 delete mode 100644 .github/workflows/old/pr.yml

diff --git a/.github/workflows/old/build_and_test.yml b/.github/workflows/old/build_and_test.yml
deleted file mode 100644
index ec917fbf092..00000000000
--- a/.github/workflows/old/build_and_test.yml
+++ /dev/null
@@ -1,74 +0,0 @@
-name: Base CCCL workflow
-
-on:
-  workflow_call:
-    inputs:
-      build_matrix: 
-        type: string
-        required: true
-      test_matrix: 
-        type: string
-        required: true
-      build_script:
-        type: string
-        required: true
-      test_script: 
-        type: string
-        required: true
-
-jobs:
-  build:
-    if: github.repository == 'NVIDIA/cccl'
-    name: Build/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.cpu}}
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(inputs.build_matrix) }}
-    runs-on: linux-${{ matrix.cpu }}-cpu16
-    container: # Fetch the container image to use for the job
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-    permissions: # Set permissions needed for sccache/AWS credentials
-      id-token: write
-      contents: read
-    steps:
-    - name: Checkout repo
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        persist-credentials: false
-    - name: Configure credentials and environment variables for sccache
-      uses: ./.github/actions/configure_cccl_sccache
-    - name: Run build script
-      run: |
-        ./${{ inputs.build_script }}
-        sccache -s
-  test:
-    needs: build
-    if: github.repository == 'NVIDIA/cccl'
-    name: Test/CUDA ${{matrix.cuda}}/${{ matrix.compiler.name }}${{ matrix.compiler.version }}/C++${{matrix.std}}/${{matrix.gpu}}/${{matrix.cpu}})
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(inputs.test_matrix) }}
-    runs-on: linux-${{matrix.cpu}}-gpu-${{matrix.gpu}}-latest-1
-    container: # Fetch the container image to use for the job
-      options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-ubuntu${{ matrix.ubuntu }}
-      env:
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-    permissions: # Set permissions needed for sccache/AWS credentials
-      id-token: write
-      contents: read
-    steps:
-    - name: Checkout repo
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        persist-credentials: false
-    - name: Configure credentials and environment variables for sccache
-      uses: ./.github/actions/configure_cccl_sccache
-    - name: Run test script
-      run: |
-        ./${{ inputs.test_script }}
-        sccache -s
diff --git a/.github/workflows/old/per_compiler.yml b/.github/workflows/old/per_compiler.yml
deleted file mode 100644
index a1335a808dc..00000000000
--- a/.github/workflows/old/per_compiler.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Per compiler
-
-on:
-  workflow_call:
-    inputs:
-      cuda_version: {type: string, required: true}
-      full_matrix: {type: string, required: true}
-      compiler_name: {type: string, required: true}
-
-jobs:
-  compute_per_compiler_matrix:
-    name: Compute ${{inputs.compiler_name}} matrix  
-    runs-on: ubuntu-latest
-    outputs:
-      PER_COMPILER_MATRIX:  ${{ steps.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX }}
-    steps:
-      - name: Compute Matrix
-        id: compute_per_compiler_matrix
-        run: |
-          FULL_MATRIX=${{ toJSON(inputs.full_matrix) }}
-          PER_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c '[.[] | select(.compiler.name == "${{ inputs.compiler_name }}" and .cuda == "${{ inputs.cuda_version }}")]')
-          echo "PER_COMPILER_MATRIX=$PER_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
-
-  per_compiler:
-    needs: compute_per_compiler_matrix
-    runs-on: ubuntu-latest
-    strategy:
-      matrix: 
-        include: ${{ fromJSON(needs.compute_per_compiler_matrix.outputs.PER_COMPILER_MATRIX) }}
-    steps:
-      - name: test
-        run: |
-          echo "CUDA: ${{matrix.cuda_version}} Compiler: ${{ matrix.compiler.name }} ${{ matrix.compiler.version }}"
-
-
diff --git a/.github/workflows/old/pr.yml b/.github/workflows/old/pr.yml
deleted file mode 100644
index 94844d412dc..00000000000
--- a/.github/workflows/old/pr.yml
+++ /dev/null
@@ -1,124 +0,0 @@
-# This is the main workflow that runs on every PR and push to main
-name: pr
-
-defaults:
-  run:
-    shell: bash
-
-on:
-  push:
-    branches:
-      - main
-#      - "pull-request/[0-9]+"
-
-# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts.
-concurrency:
-  group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }}
-  cancel-in-progress: true
-
-jobs:
-  compute-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      BUILD_MATRIX: ${{ steps.compute-matrix.outputs.BUILD_MATRIX }}
-      TEST_MATRIX:  ${{ steps.compute-matrix.outputs.TEST_MATRIX }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Compute matrix
-        id: compute-matrix
-        run: |
-          cat ./ci/matrix.yaml
-          BUILD_MATRIX=$(yq -o=json ./ci/matrix.yaml | jq -c '[.pull_request[] | . as $o | {std: .std[]} + del($o.std)] | {include: . }' )
-          echo "BUILD_MATRIX=$BUILD_MATRIX" | tee -a "$GITHUB_OUTPUT"
-          GPUS=$(yq -o=json ./ci/matrix.yaml | jq '.gpus')
-          TEST_MATRIX=$(echo $BUILD_MATRIX | jq -c --argjson gpus "$GPUS" '[.include, $gpus] | [ combinations | .[0] + {gpu: .[1]} ] | {include: . }')
-          echo "TEST_MATRIX=$TEST_MATRIX" | tee -a "$GITHUB_OUTPUT"
-
-  thrust:
-    needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
-    with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
-
-  libcudacxx:
-    needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
-    with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_libcudacxx.sh"
-      test_script: "./ci/test_libcudacxx.sh"
-
-  cub:
-    needs: compute-matrix
-    uses: ./.github/workflows/build_and_test.yml
-    with:
-      build_matrix: ${{ needs.compute-matrix.outputs.BUILD_MATRIX }}
-      test_matrix: ${{ needs.compute-matrix.outputs.TEST_MATRIX }}
-      build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
-
-    #steps:
-#   #   - name: Verify compiler version
-#   #     run: |
-#   #       version=$("${{matrix.compiler.exe}}" --version | head -n1 | cut -d' ' -f3 | cut -d. -f1)
-#   #       if [[ "${version}" != "${{ matrix.compiler.version }}" ]]; then
-#   #         echo "Compiler version does not match: found ${version}, expected ${{ matrix.compiler.version }}"
-#   #         exit 1
-#   #       fi
-#   #       echo "Sucessfully found ${{matrix.compiler.name}} ${version}"
-    #  - name: Get AWS credentials for sccache bucket
-    #    uses: aws-actions/configure-aws-credentials@v2
-    #    with:
-    #      role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-NVIDIA
-    #      aws-region: us-east-2 
-    #      role-duration-seconds: 43200 # 12 hours
-    #  - uses: actions/checkout@v3
-    #    with:
-    #      fetch-depth: 0
-    #      persist-credentials: false
-    #  - name: Build Thrust tests
-    #    env:
-    #      SCCACHE_BUCKET: rapids-sccache-east
-    #      SCCACHE_REGION: us-east-2
-    #      SCCACHE_IDLE_TIMEOUT: 32768
-    #      SCCACHE_S3_USE_SSL: true
-    #      SCCACHE_S3_NO_CREDENTIALS: false
-    #    run: |
-    #      time bash ./ci/build_thrust.sh 
-    #      sccache -s
-    #  - name: tar Thrust build artifacts
-    #    run: |
-    #      tar czf thrust-build-artifacts.tar.gz ./thrust
-    #      echo "tarball size: $(du -h thrust-build-artifacts.tar.gz | cut -f1)"
-    #  - name: Upload Thrust build artifacts
-    #    uses: actions/upload-artifact@v3
-    #    with:
-    #      name: thrust-build-artifacts
-    #      path: thrust-build-artifacts.tar.gz
-  #test:
-  #  name: Run tests
-  #  needs: build
-  #  runs-on: linux-amd64-gpu-v100-latest-1
-  #  container:
-  #    options: -u root
-  #    image: rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04
-  #    env:
-  #      NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-  #  steps:
-  #    - name: Download Thrust build artifacts
-  #      uses: actions/download-artifact@v3
-  #      with:
-  #        name: thrust-build-artifacts
-  #    - name: Untar Thrust build artifacts
-  #      run: tar xzf thrust-build-artifacts.tar.gz
-  #    - name: Run Thrust tests
-  #      run: | 
-  #        pwd 
-  #        ls
-  #        chmod -R +x thrust/build/bin/*
-  #        ctest --test-dir thrust/build --output-on-failure
\ No newline at end of file

From e00d3ae6eb1da1d9ecb89a8bde8e6f4356ac66da Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 7 Jun 2023 23:55:03 +0000
Subject: [PATCH 358/395] Delete old CI scripts.

---
 ci/get_groups.sh     | 18 ------------------
 ci/group_by_field.jq |  4 ----
 2 files changed, 22 deletions(-)
 delete mode 100755 ci/get_groups.sh
 delete mode 100644 ci/group_by_field.jq

diff --git a/ci/get_groups.sh b/ci/get_groups.sh
deleted file mode 100755
index e7c0c256c51..00000000000
--- a/ci/get_groups.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# Ensure that the script is being executed from its directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
-
-# Check if input JSON and field arguments are provided
-if [ "$#" -ne 2 ]; then
-  echo "Usage: ./get_groups.sh <input_json> <field>"
-  exit 1
-fi
-
-# Assign command-line arguments to variables
-input_json="$1"
-field="$2"
-
-output=$(echo $input_json | jq -L . -c --arg field "$field" 'include "group_by_field"; group_by_field($field)')
-echo $output 
-
diff --git a/ci/group_by_field.jq b/ci/group_by_field.jq
deleted file mode 100644
index d074a0a655c..00000000000
--- a/ci/group_by_field.jq
+++ /dev/null
@@ -1,4 +0,0 @@
-# Groups an array of objects by a field and returns an object with the field value as key and the array of objects as value
-# Handles nested fields (e.g. "compiler.name")
-def group_by_field($field):
-    group_by(getpath($field | split("."))) | map( {(.[0] | getpath($field | split("."))): .}) | add;

From c57c6cc4eb8494c148670903d4a388ccfdb1f603 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 8 Jun 2023 00:19:33 +0000
Subject: [PATCH 359/395] Remove devcontainers from this branch.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json | 22 ----------
 .devcontainer/cuda11.0-gcc8/devcontainer.json | 22 ----------
 .devcontainer/cuda11.0-gcc9/devcontainer.json | 22 ----------
 .../cuda11.0-llvm9/devcontainer.json          | 22 ----------
 .../cuda12.1-gcc10/devcontainer.json          | 22 ----------
 .../cuda12.1-gcc11/devcontainer.json          | 22 ----------
 .../cuda12.1-gcc12/devcontainer.json          | 22 ----------
 .devcontainer/cuda12.1-gcc7/devcontainer.json | 22 ----------
 .devcontainer/cuda12.1-gcc8/devcontainer.json | 22 ----------
 .devcontainer/cuda12.1-gcc9/devcontainer.json | 22 ----------
 .../cuda12.1-llvm10/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm11/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm12/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm13/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm14/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm15/devcontainer.json         | 22 ----------
 .../cuda12.1-llvm16/devcontainer.json         | 21 ----------
 .../cuda12.1-llvm9/devcontainer.json          | 22 ----------
 .devcontainer/devcontainer.json               | 20 ---------
 .devcontainer/make_devcontainers.sh           | 42 -------------------
 20 files changed, 457 deletions(-)
 delete mode 100644 .devcontainer/cuda11.0-gcc7/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-gcc8/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-gcc9/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-llvm9/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc10/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc11/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc12/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc7/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc8/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-gcc9/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm10/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm11/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm12/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm13/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm14/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm15/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm16/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm9/devcontainer.json
 delete mode 100644 .devcontainer/devcontainer.json
 delete mode 100755 .devcontainer/make_devcontainers.sh

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
deleted file mode 100644
index 976ad3d8666..00000000000
--- a/.devcontainer/cuda11.0-gcc7/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc7"
-}
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
deleted file mode 100644
index ccf2707b6c6..00000000000
--- a/.devcontainer/cuda11.0-gcc8/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc8"
-}
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
deleted file mode 100644
index 8063d298f00..00000000000
--- a/.devcontainer/cuda11.0-gcc9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc9"
-}
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
deleted file mode 100644
index 52946df942c..00000000000
--- a/.devcontainer/cuda11.0-llvm9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-llvm9"
-}
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
deleted file mode 100644
index c8d8d19c4f3..00000000000
--- a/.devcontainer/cuda12.1-gcc10/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc10"
-}
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
deleted file mode 100644
index 5d31a63dfc0..00000000000
--- a/.devcontainer/cuda12.1-gcc11/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc11"
-}
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
deleted file mode 100644
index f66384fc7d8..00000000000
--- a/.devcontainer/cuda12.1-gcc12/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc12"
-}
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
deleted file mode 100644
index b6b6f982949..00000000000
--- a/.devcontainer/cuda12.1-gcc7/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc7"
-}
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
deleted file mode 100644
index 6fffb8aff1f..00000000000
--- a/.devcontainer/cuda12.1-gcc8/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc8"
-}
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
deleted file mode 100644
index a68bc776134..00000000000
--- a/.devcontainer/cuda12.1-gcc9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-gcc9"
-}
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
deleted file mode 100644
index 10307b85907..00000000000
--- a/.devcontainer/cuda12.1-llvm10/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm10"
-}
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
deleted file mode 100644
index 5bda646de84..00000000000
--- a/.devcontainer/cuda12.1-llvm11/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm11"
-}
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
deleted file mode 100644
index 748aec1361d..00000000000
--- a/.devcontainer/cuda12.1-llvm12/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm12"
-}
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
deleted file mode 100644
index 2b4652a454d..00000000000
--- a/.devcontainer/cuda12.1-llvm13/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm13"
-}
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
deleted file mode 100644
index eb4eee24b2b..00000000000
--- a/.devcontainer/cuda12.1-llvm14/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm14"
-}
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
deleted file mode 100644
index 8bbcfaf933a..00000000000
--- a/.devcontainer/cuda12.1-llvm15/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm15"
-}
diff --git a/.devcontainer/cuda12.1-llvm16/devcontainer.json b/.devcontainer/cuda12.1-llvm16/devcontainer.json
deleted file mode 100644
index d020ee58aa3..00000000000
--- a/.devcontainer/cuda12.1-llvm16/devcontainer.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
-}
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
deleted file mode 100644
index d971fff7795..00000000000
--- a/.devcontainer/cuda12.1-llvm9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda12.1-llvm9"
-}
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
deleted file mode 100644
index b60dd7cfc78..00000000000
--- a/.devcontainer/devcontainer.json
+++ /dev/null
@@ -1,20 +0,0 @@
-
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
-
-  "hostRequirements": { "gpu": true },
-
-  "initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,config}"],
-
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
-}
\ No newline at end of file
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
deleted file mode 100755
index 7e5e430cfc5..00000000000
--- a/.devcontainer/make_devcontainers.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-# This script parses the CI matrix.yaml file and generates a devcontainer.json file for each unique combination of 
-# CUDA version, compiler name/version, and Ubuntu version. The devcontainer.json files are written to the
-# .devcontainer directory to a subdirectory named after the CUDA version and compiler name/version.
-# GitHub docs on using multiple devcontainer.json files:
-# https://docs.github.com/en/codespaces/setting-up-your-project-for-codespaces/adding-a-dev-container-configuration/introduction-to-dev-containers#devcontainerjson
-
-# Ensure the script is being executed in its containing directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
-
-# The root devcontainer.json file is used as a template for all other devcontainer.json files
-# by replacing the `image:` field with the appropriate image name
-base_devcontainer_file="./devcontainer.json"
-
-# Define image root
-IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"
-
-# Read matrix.yaml and convert it to json
-matrix_json=$(yq -o json ../ci/matrix.yaml)
-
-# Get unique combinations of cuda version, compiler name/version, and Ubuntu version
-combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, ubuntu: .ubuntu}] | unique | .[]')
-
-# For each unique combination
-for combination in $combinations; do
-    cuda_version=$(echo "$combination" | jq -r '.cuda')
-    compiler_name=$(echo "$combination" | jq -r '.compiler_name')
-    compiler_version=$(echo "$combination" | jq -r '.compiler_version')
-    ubuntu_version=$(echo "$combination" | jq -r '.ubuntu')
-
-    name="cuda$cuda_version-$compiler_name$compiler_version"
-    mkdir -p "$name"
-    devcontainer_file="$name/devcontainer.json"
-    image="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
-
-    # Use the base_devcontainer.json as a template, plug in the CUDA, compiler names, versions, and Ubuntu version,
-    # and write the output to the new devcontainer.json file
-    jq --arg image "$image"  --arg name "$name" '. + {image: $image, name: $name}' $base_devcontainer_file > "$devcontainer_file"
-
-    echo "Created $devcontainer_file"
-done

From d641186532abb3d00ce3f4c3221936db441f1d80 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 8 Jun 2023 00:21:28 +0000
Subject: [PATCH 360/395] Add devcontainers.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json | 22 ++++++++++
 .devcontainer/cuda11.0-gcc8/devcontainer.json | 22 ++++++++++
 .devcontainer/cuda11.0-gcc9/devcontainer.json | 22 ++++++++++
 .../cuda11.0-llvm9/devcontainer.json          | 22 ++++++++++
 .../cuda12.1-gcc10/devcontainer.json          | 22 ++++++++++
 .../cuda12.1-gcc11/devcontainer.json          | 22 ++++++++++
 .../cuda12.1-gcc12/devcontainer.json          | 22 ++++++++++
 .devcontainer/cuda12.1-gcc7/devcontainer.json | 22 ++++++++++
 .devcontainer/cuda12.1-gcc8/devcontainer.json | 22 ++++++++++
 .devcontainer/cuda12.1-gcc9/devcontainer.json | 22 ++++++++++
 .../cuda12.1-llvm10/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm11/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm12/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm13/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm14/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm15/devcontainer.json         | 22 ++++++++++
 .../cuda12.1-llvm16/devcontainer.json         | 21 ++++++++++
 .../cuda12.1-llvm9/devcontainer.json          | 22 ++++++++++
 .devcontainer/devcontainer.json               | 20 +++++++++
 .devcontainer/make_devcontainers.sh           | 42 +++++++++++++++++++
 20 files changed, 457 insertions(+)
 create mode 100644 .devcontainer/cuda11.0-gcc7/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-gcc8/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-gcc9/devcontainer.json
 create mode 100644 .devcontainer/cuda11.0-llvm9/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc10/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc11/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc12/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc7/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc8/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-gcc9/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm10/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm11/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm12/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm13/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm14/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm15/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm16/devcontainer.json
 create mode 100644 .devcontainer/cuda12.1-llvm9/devcontainer.json
 create mode 100644 .devcontainer/devcontainer.json
 create mode 100755 .devcontainer/make_devcontainers.sh

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
new file mode 100644
index 00000000000..976ad3d8666
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc7/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.0-gcc7"
+}
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
new file mode 100644
index 00000000000..ccf2707b6c6
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc8/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.0-gcc8"
+}
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
new file mode 100644
index 00000000000..8063d298f00
--- /dev/null
+++ b/.devcontainer/cuda11.0-gcc9/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.0-gcc9"
+}
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
new file mode 100644
index 00000000000..52946df942c
--- /dev/null
+++ b/.devcontainer/cuda11.0-llvm9/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.0-llvm9"
+}
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
new file mode 100644
index 00000000000..c8d8d19c4f3
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc10"
+}
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
new file mode 100644
index 00000000000..5d31a63dfc0
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc11"
+}
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
new file mode 100644
index 00000000000..f66384fc7d8
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc12"
+}
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
new file mode 100644
index 00000000000..b6b6f982949
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc7"
+}
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
new file mode 100644
index 00000000000..6fffb8aff1f
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc8"
+}
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
new file mode 100644
index 00000000000..a68bc776134
--- /dev/null
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-gcc9"
+}
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
new file mode 100644
index 00000000000..10307b85907
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm10"
+}
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
new file mode 100644
index 00000000000..5bda646de84
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm11"
+}
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
new file mode 100644
index 00000000000..748aec1361d
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm12"
+}
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
new file mode 100644
index 00000000000..2b4652a454d
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm13"
+}
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
new file mode 100644
index 00000000000..eb4eee24b2b
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm14"
+}
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
new file mode 100644
index 00000000000..8bbcfaf933a
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm15"
+}
diff --git a/.devcontainer/cuda12.1-llvm16/devcontainer.json b/.devcontainer/cuda12.1-llvm16/devcontainer.json
new file mode 100644
index 00000000000..d020ee58aa3
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm16/devcontainer.json
@@ -0,0 +1,21 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ]
+}
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
new file mode 100644
index 00000000000..d971fff7795
--- /dev/null
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -0,0 +1,22 @@
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda12.1-llvm9"
+}
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 00000000000..b60dd7cfc78
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,20 @@
+
+{
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
+
+  "hostRequirements": { "gpu": true },
+
+  "initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,config}"],
+
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+
+  "mounts": [
+    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ]
+}
\ No newline at end of file
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
new file mode 100755
index 00000000000..7e5e430cfc5
--- /dev/null
+++ b/.devcontainer/make_devcontainers.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# This script parses the CI matrix.yaml file and generates a devcontainer.json file for each unique combination of 
+# CUDA version, compiler name/version, and Ubuntu version. The devcontainer.json files are written to the
+# .devcontainer directory to a subdirectory named after the CUDA version and compiler name/version.
+# GitHub docs on using multiple devcontainer.json files:
+# https://docs.github.com/en/codespaces/setting-up-your-project-for-codespaces/adding-a-dev-container-configuration/introduction-to-dev-containers#devcontainerjson
+
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+# The root devcontainer.json file is used as a template for all other devcontainer.json files
+# by replacing the `image:` field with the appropriate image name
+base_devcontainer_file="./devcontainer.json"
+
+# Define image root
+IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"
+
+# Read matrix.yaml and convert it to json
+matrix_json=$(yq -o json ../ci/matrix.yaml)
+
+# Get unique combinations of cuda version, compiler name/version, and Ubuntu version
+combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, ubuntu: .ubuntu}] | unique | .[]')
+
+# For each unique combination
+for combination in $combinations; do
+    cuda_version=$(echo "$combination" | jq -r '.cuda')
+    compiler_name=$(echo "$combination" | jq -r '.compiler_name')
+    compiler_version=$(echo "$combination" | jq -r '.compiler_version')
+    ubuntu_version=$(echo "$combination" | jq -r '.ubuntu')
+
+    name="cuda$cuda_version-$compiler_name$compiler_version"
+    mkdir -p "$name"
+    devcontainer_file="$name/devcontainer.json"
+    image="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
+
+    # Use the base_devcontainer.json as a template, plug in the CUDA, compiler names, versions, and Ubuntu version,
+    # and write the output to the new devcontainer.json file
+    jq --arg image "$image"  --arg name "$name" '. + {image: $image, name: $name}' $base_devcontainer_file > "$devcontainer_file"
+
+    echo "Created $devcontainer_file"
+done

From 17c74a1320feda9fd06c0114a78eb0082d08802e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 8 Jun 2023 04:45:06 +0000
Subject: [PATCH 361/395] Make thrust/cub tests timeout after 15 seconds.

---
 ci/test_cub.sh    | 2 +-
 ci/test_thrust.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/test_cub.sh b/ci/test_cub.sh
index 20c82b8af24..74beba45bb8 100755
--- a/ci/test_cub.sh
+++ b/ci/test_cub.sh
@@ -7,6 +7,6 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
 ./build_cub.sh "$@"
 
-ctest --test-dir ../build --output-on-failure
+ctest --test-dir ../build --output-on-failure --timeout 15
 
 echo "CUB test complete"
\ No newline at end of file
diff --git a/ci/test_thrust.sh b/ci/test_thrust.sh
index d38cf5f706c..cec1deca519 100755
--- a/ci/test_thrust.sh
+++ b/ci/test_thrust.sh
@@ -7,7 +7,7 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
 
 ./build_thrust.sh "$@"
 
-ctest --test-dir ../build --output-on-failure
+ctest --test-dir ../build --output-on-failure --timeout 15
 
 echo "Thrust test complete"
 

From 8177c03057ebae3581ea31d535fab16960fc51b2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 19:20:23 +0000
Subject: [PATCH 362/395] Update permissions according to AJ's suggestion.

---
 .github/workflows/build-and-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b7767d2f85f..562c7e7b7d3 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -28,7 +28,6 @@ jobs:
       image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
     permissions:
       id-token: write
-      contents: read
     steps:
       - name: Echo git version
         run: git --version
@@ -67,7 +66,6 @@ jobs:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
     permissions:
       id-token: write
-      contents: read
     steps:
     # TODO: We can remove this when we aren't using submodules anymore
       - name: Update git version

From 1c16b96e333ba261b6d706b623975a15ae965ad8 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 19:24:27 +0000
Subject: [PATCH 363/395] Add .cache to gitignore.

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index aec8d8b240b..2d7da49b38e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .idea/
 *build*/
+.cache
\ No newline at end of file

From 6424b79558540ae98b3a248a6d5cccf29746866c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:22:32 +0000
Subject: [PATCH 364/395] Add overview of CI pipeline.

---
 README.md | 43 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 40 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 13865022003..403350cc388 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,42 @@
-# CUDA C++ Core Libraries (CCCL)
+## CI Pipeline Overview
 
-The future home of the CUDA C++ Core Libraries (Thrust, CUB, libcu++).
+Welcome to the CI pipeline for CCCL! In this document, we will provide you with a high-level overview of the Continuous Integration (CI) process using GitHub Actions. The CI pipeline helps us automate the building, testing, and validation of our project to ensure its quality and stability.
 
-Stay tuned. 
+### Workflow Structure
+
+Our CI pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:
+
+#### [pr.yml](.github/workflows/pr.yml)
+
+This workflow runs on every pull request and push to the `main` branch. It orchestrates the entire CI process and includes the following jobs:
+
+- [`compute-matrix`](#compute-matrix-yml): Reads the matrix configuration from the `matrix.yaml` file and computes the necessary build configurations based on the PR or push event.
+- `thrust`, `cub`, and `libcudacxx`: Build and test jobs for each CCCL component. They are executed in parallel for each configuration specified in the matrix. These jobs are optional and depend on the presence of the corresponding build and test scripts.
+- `ci`: A final job that runs after all other jobs and performs branch protection status checks.
+
+#### [compute-matrix.yml](.github/workflows/compute-matrix.yml)
+
+The matrix configuration defines the various combinations of build configurations we want to test. It is specified in the `matrix.yaml` file. The configurations include:
+
+- `cuda`: The CUDA Toolkit version.
+- `os`: The operating system.
+- `cpu`: The CPU architecture.
+- `compiler`: The compiler to use, including its name, version, and executable.
+- `gpu_build_archs`: The GPU architectures to build for, separated by semicolons.
+- `std`: The C++ standards to build for. Each value generates an independent build/test job.
+
+The `compute-matrix.yml` workflow reads this matrix configuration and computes the necessary combinations based on the matrix type (e.g., "pull-request"). The computed matrix is used by other workflows to determine the configurations to build and test.
+
+#### [dispatch-build-and-test.yml](#dispatch-build-and-test-yml)
+
+This workflow is called by the `thrust`, `cub`, and `libcudacxx` jobs from the [`pr.yml`](#pr.yml) workflow. It dispatches the [`build-and-test.yml`](#build-and-test.yml) workflow for each specific build configuration in parallel. It includes the following job:
+
+- `build_and_test`: Dispatches the `build-and-test.yml` workflow for each configuration. It provides the necessary inputs, such as the compiler, CUDA version, and build/test scripts, to the dispatched workflow.
+
+Dispatching the `build-and-test.yml` workflow for each specific configuration allows us to execute the build and test steps concurrently. This parallel execution reduces the overall execution time and improves the efficiency of our CI pipeline.
+#### [build-and-test.yml](.github/workflows/build-and-test.yml)
+
+This reusable workflow is called by the `thrust`, `cub`, and `libcudacxx` jobs from the `pr.yml` workflow. It contains two main jobs:
+
+- `build`: Builds the project using the specified build script and the given compiler, CUDA version, and other build configurations. This job is optional and depends on the presence of the build script.
+- `test`: Runs the tests using the specified test script and the same configurations as the build job. This job is optional and depends on the presence of the test script.

From 44dee4fc3560ddf666bd28e809341a0717db8274 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:29:00 +0000
Subject: [PATCH 365/395] Add mermaid diagram.

---
 README.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/README.md b/README.md
index 403350cc388..4b0ad659c86 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,26 @@ Welcome to the CI pipeline for CCCL! In this document, we will provide you with
 
 ### Workflow Structure
 
+```mermaid
+graph LR
+  A[On Push to PR] --> D(Compute Matrix)
+  D --> E{Dispatch Build and Test}
+  E --> F(Build Job)
+  E --> G(Test Job)
+  F --> H{Build Script Provided?}
+  G --> I{Test Script Provided?}
+  H -- Yes --> J[Build Completed]
+  I -- Yes --> K[Test Completed]
+  J --> L{All Builds Completed?}
+  K --> L
+  L --> M{All Tests Completed?}
+  M --> N{All Jobs Succeeded?}
+  N --> O{CI Success}
+  O --> P[Workflow Completed]
+  H -- No --> L
+  I -- No --> M
+```
+
 Our CI pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:
 
 #### [pr.yml](.github/workflows/pr.yml)

From ceb7fe715db30ffe57f2fe83a5b8c1b86e5735ee Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:43:25 +0000
Subject: [PATCH 366/395] Better diagram

---
 README.md | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 4b0ad659c86..6945a39bf10 100644
--- a/README.md
+++ b/README.md
@@ -5,23 +5,17 @@ Welcome to the CI pipeline for CCCL! In this document, we will provide you with
 ### Workflow Structure
 
 ```mermaid
-graph LR
-  A[On Push to PR] --> D(Compute Matrix)
-  D --> E{Dispatch Build and Test}
-  E --> F(Build Job)
-  E --> G(Test Job)
-  F --> H{Build Script Provided?}
-  G --> I{Test Script Provided?}
-  H -- Yes --> J[Build Completed]
-  I -- Yes --> K[Test Completed]
-  J --> L{All Builds Completed?}
-  K --> L
-  L --> M{All Tests Completed?}
-  M --> N{All Jobs Succeeded?}
-  N --> O{CI Success}
-  O --> P[Workflow Completed]
-  H -- No --> L
-  I -- No --> M
+graph TD
+  A[Event] --> D(Compute Matrix)
+  D --> E{Dispatch Build and Test \n for each configuration}
+  E --> F1(Build Job)
+  F1 --> H1{Run Build Script}
+  H1 --> J1[Build Completed]
+  J1 --> G1(Test Job)
+  G1 --> I1{Run Test Script}
+  I1 --> L1[Test Completed]
+  L1 --> O{CI Success}
+  E -- For each configuration... --> O
 ```
 
 Our CI pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:

From a929f5b4c7aad2cf93b6dae3c65ca230657f4553 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:45:04 +0000
Subject: [PATCH 367/395] Update readme.

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6945a39bf10..31b04814844 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,8 @@ Welcome to the CI pipeline for CCCL! In this document, we will provide you with
 
 ### Workflow Structure
 
+Our CI workflow can be best summarized by this diagram:
+
 ```mermaid
 graph TD
   A[Event] --> D(Compute Matrix)
@@ -18,7 +20,7 @@ graph TD
   E -- For each configuration... --> O
 ```
 
-Our CI pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:
+This pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:
 
 #### [pr.yml](.github/workflows/pr.yml)
 

From 20c77786b39bac5eccf795f95fa537e2aaea1116 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:47:54 +0000
Subject: [PATCH 368/395] Fix link.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 31b04814844..576555e3523 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ The matrix configuration defines the various combinations of build configuration
 
 The `compute-matrix.yml` workflow reads this matrix configuration and computes the necessary combinations based on the matrix type (e.g., "pull-request"). The computed matrix is used by other workflows to determine the configurations to build and test.
 
-#### [dispatch-build-and-test.yml](#dispatch-build-and-test-yml)
+#### [dispatch-build-and-test.yml](.github/workflows/dispatch-build-and-test.yml)
 
 This workflow is called by the `thrust`, `cub`, and `libcudacxx` jobs from the [`pr.yml`](#pr.yml) workflow. It dispatches the [`build-and-test.yml`](#build-and-test.yml) workflow for each specific build configuration in parallel. It includes the following job:
 

From 1c451aa5c39f6c240976d6feaa686735270a8dd0 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Tue, 20 Jun 2023 20:49:10 +0000
Subject: [PATCH 369/395] Add links to matrix file.

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 576555e3523..3badd170524 100644
--- a/README.md
+++ b/README.md
@@ -26,13 +26,13 @@ This pipeline consists of multiple workflows, each serving a specific purpose. L
 
 This workflow runs on every pull request and push to the `main` branch. It orchestrates the entire CI process and includes the following jobs:
 
-- [`compute-matrix`](#compute-matrix-yml): Reads the matrix configuration from the `matrix.yaml` file and computes the necessary build configurations based on the PR or push event.
+- [`compute-matrix`](#compute-matrix-yml): Reads the matrix configuration from the [`matrix.yaml`](ci/matrix.yaml) file and computes the necessary build configurations based on the PR or push event.
 - `thrust`, `cub`, and `libcudacxx`: Build and test jobs for each CCCL component. They are executed in parallel for each configuration specified in the matrix. These jobs are optional and depend on the presence of the corresponding build and test scripts.
 - `ci`: A final job that runs after all other jobs and performs branch protection status checks.
 
 #### [compute-matrix.yml](.github/workflows/compute-matrix.yml)
 
-The matrix configuration defines the various combinations of build configurations we want to test. It is specified in the `matrix.yaml` file. The configurations include:
+The matrix configuration defines the various combinations of build configurations we want to test. It is specified in the [`matrix.yaml`](ci/matrix.yaml) file. The configurations include:
 
 - `cuda`: The CUDA Toolkit version.
 - `os`: The operating system.

From 9b66bf5eec07f9eb946e88909e043b0b0bd7be60 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 14:59:42 +0000
Subject: [PATCH 370/395] Update CI doc and link to it from readme.

---
 README.md      | 59 ++++------------------------------------------
 ci-overview.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 54 deletions(-)
 create mode 100644 ci-overview.md

diff --git a/README.md b/README.md
index 3badd170524..e15d6e2a76e 100644
--- a/README.md
+++ b/README.md
@@ -1,58 +1,9 @@
-## CI Pipeline Overview
-
-Welcome to the CI pipeline for CCCL! In this document, we will provide you with a high-level overview of the Continuous Integration (CI) process using GitHub Actions. The CI pipeline helps us automate the building, testing, and validation of our project to ensure its quality and stability.
-
-### Workflow Structure
-
-Our CI workflow can be best summarized by this diagram:
-
-```mermaid
-graph TD
-  A[Event] --> D(Compute Matrix)
-  D --> E{Dispatch Build and Test \n for each configuration}
-  E --> F1(Build Job)
-  F1 --> H1{Run Build Script}
-  H1 --> J1[Build Completed]
-  J1 --> G1(Test Job)
-  G1 --> I1{Run Test Script}
-  I1 --> L1[Test Completed]
-  L1 --> O{CI Success}
-  E -- For each configuration... --> O
-```
-
-This pipeline consists of multiple workflows, each serving a specific purpose. Let's go through each of them:
-
-#### [pr.yml](.github/workflows/pr.yml)
-
-This workflow runs on every pull request and push to the `main` branch. It orchestrates the entire CI process and includes the following jobs:
+# CUDA C++ Core Libraries (CCCL)
 
-- [`compute-matrix`](#compute-matrix-yml): Reads the matrix configuration from the [`matrix.yaml`](ci/matrix.yaml) file and computes the necessary build configurations based on the PR or push event.
-- `thrust`, `cub`, and `libcudacxx`: Build and test jobs for each CCCL component. They are executed in parallel for each configuration specified in the matrix. These jobs are optional and depend on the presence of the corresponding build and test scripts.
-- `ci`: A final job that runs after all other jobs and performs branch protection status checks.
+The future home of the CUDA C++ Core Libraries (Thrust, CUB, libcu++).
 
-#### [compute-matrix.yml](.github/workflows/compute-matrix.yml)
+Stay tuned. 
 
-The matrix configuration defines the various combinations of build configurations we want to test. It is specified in the [`matrix.yaml`](ci/matrix.yaml) file. The configurations include:
-
-- `cuda`: The CUDA Toolkit version.
-- `os`: The operating system.
-- `cpu`: The CPU architecture.
-- `compiler`: The compiler to use, including its name, version, and executable.
-- `gpu_build_archs`: The GPU architectures to build for, separated by semicolons.
-- `std`: The C++ standards to build for. Each value generates an independent build/test job.
-
-The `compute-matrix.yml` workflow reads this matrix configuration and computes the necessary combinations based on the matrix type (e.g., "pull-request"). The computed matrix is used by other workflows to determine the configurations to build and test.
-
-#### [dispatch-build-and-test.yml](.github/workflows/dispatch-build-and-test.yml)
-
-This workflow is called by the `thrust`, `cub`, and `libcudacxx` jobs from the [`pr.yml`](#pr.yml) workflow. It dispatches the [`build-and-test.yml`](#build-and-test.yml) workflow for each specific build configuration in parallel. It includes the following job:
-
-- `build_and_test`: Dispatches the `build-and-test.yml` workflow for each configuration. It provides the necessary inputs, such as the compiler, CUDA version, and build/test scripts, to the dispatched workflow.
-
-Dispatching the `build-and-test.yml` workflow for each specific configuration allows us to execute the build and test steps concurrently. This parallel execution reduces the overall execution time and improves the efficiency of our CI pipeline.
-#### [build-and-test.yml](.github/workflows/build-and-test.yml)
-
-This reusable workflow is called by the `thrust`, `cub`, and `libcudacxx` jobs from the `pr.yml` workflow. It contains two main jobs:
+## CI Pipeline Overview
 
-- `build`: Builds the project using the specified build script and the given compiler, CUDA version, and other build configurations. This job is optional and depends on the presence of the build script.
-- `test`: Runs the tests using the specified test script and the same configurations as the build job. This job is optional and depends on the presence of the test script.
+For a detailed overview of the CI pipeline, see [ci-overview.md](ci-overview.md).
\ No newline at end of file
diff --git a/ci-overview.md b/ci-overview.md
new file mode 100644
index 00000000000..59e818b424c
--- /dev/null
+++ b/ci-overview.md
@@ -0,0 +1,63 @@
+# CCCL - Continuous Integration (CI) Workflow 
+As a header-only C++ library, the code we write isn't compiled until a developer includes it in their project. 
+
+This means our code needs to be robust enough to compile and run across a variety of platforms, compilers, and configurations. 
+As such, maintaining extensive and thorough Continuous Integration (CI) coverage is crucial. 
+
+Our CI system needs to test various combinations of operating systems, CPU architectures, compilers, and C++ standards. 
+The number of configurations can be quite large and hence our CI system must be designed to handle this complexity. 
+We use GitHub Actions for this purpose. It is flexible, powerful, and well-integrated with GitHub.
+
+This document will walk you through our CI workflow and guide you on how to interact with and troubleshoot it when needed.
+
+## TL;DR - Workflow Overview
+
+```mermaid
+graph LR
+    A[Workflow Starts on Push to PR]
+    A --> B{Computes Matrix}
+    B --> C[For Each Matrix Configuration]
+    C --> D{Runs a Build Job}
+    D --> E[Build Job Completed]
+    E --> F{Runs a Test Job}
+    F --> G[Test Job Completed]
+    G --> H[CI Successful]
+```
+
+This repository relies on a GitHub Actions-based Continuous Integration (CI) workflow. Here's what you need to know:
+
+- **Trigger:** The main workflow triggers on every push to the main branch or pull request (PR).
+- **Execution:** The workflow generates a matrix of build configurations, based on the settings in matrix.yml, and then dispatches separate build and test jobs for each configuration. 
+- **Failures:** If a job fails, you'll be notified through GitHub's interface. You can then check the logs for details.
+- **Recovery:** To handle job failures, pull the relevant container image and rerun the script locally to reproduce the issue.
+
+## The Matrix
+
+The matrix defined in the [`matrix.yml`](ci/matrix.yaml) is the single source of truth for the environments we test our code against.
+It dictates the build configurations, such as CUDA version, operating system, CPU architecture, compiler, GPU architectures, and C++ standards. 
+It allows us to test our code against different combinations of these variables to ensure our code's compatibility and stability. 
+
+## Lifecycle of a Pull Request
+
+From creation to merging, a pull request in this project follows these steps:
+
+1. Create a PR: Once you make a change, open a PR. 
+  - If you have write permission to the repository, CI workflow will automatically start.
+  - If you don't have write permission, the workflow will start once a maintainer comments on the PR with `/ok to test`. This comment is required for all subsequent workflow runs.
+2. Wait for results: GitHub Actions executes the defined CI workflow, running jobs based on the matrix configuration.
+3. Interpret results: Check the status of the workflow. If it passes, all tests have passed on all defined configurations, and your changes likely didn't break anything. 
+4. Handle failures: If any job fails, the logs will provide information on what went wrong. 
+5. Rerun jobs: If the failure seems unrelated to your changes (e.g., due to a temporary external issue), you can rerun the jobs.
+
+## Troubleshooting Guide
+
+If a CI job fails, here's what you can do to troubleshoot:
+
+1. Check the logs: The logs provide detailed information on what went wrong during the execution. This is your starting point.
+2. Reproduce the issue locally: Pull the relevant container image and rerun the script that failed. This will allow you to dig into the issue in depth.
+3. Fix the issue: Once you've identified the problem, you can make appropriate changes to your code and rerun the CI jobs.
+
+You can refer to [GitHub Actions documentation](https://docs.github.com/en/actions) for a deeper understanding of the process and the [GitHub Actions workflows syntax](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions) to comprehend the workflow files' structure and syntax.
+
+You're not in this alone - the community is here to help. If you're stuck, don't hesitate to raise an issue or ask for assistance. Open source thrives on collaboration and learning. Happy coding!
+

From 93b3de48a856fbf41a2ca29992dbe349d3ede7a0 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 15:00:59 +0000
Subject: [PATCH 371/395] Fix bullet indentation.

---
 ci-overview.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci-overview.md b/ci-overview.md
index 59e818b424c..47f71ea9dd5 100644
--- a/ci-overview.md
+++ b/ci-overview.md
@@ -42,8 +42,8 @@ It allows us to test our code against different combinations of these variables
 From creation to merging, a pull request in this project follows these steps:
 
 1. Create a PR: Once you make a change, open a PR. 
-  - If you have write permission to the repository, CI workflow will automatically start.
-  - If you don't have write permission, the workflow will start once a maintainer comments on the PR with `/ok to test`. This comment is required for all subsequent workflow runs.
+    - If you have write permission to the repository, CI workflow will automatically start.
+    - If you don't have write permission, the workflow will start once a maintainer comments on the PR with `/ok to test`. This comment is required for all subsequent workflow runs.
 2. Wait for results: GitHub Actions executes the defined CI workflow, running jobs based on the matrix configuration.
 3. Interpret results: Check the status of the workflow. If it passes, all tests have passed on all defined configurations, and your changes likely didn't break anything. 
 4. Handle failures: If any job fails, the logs will provide information on what went wrong. 

From e5e148b2f6b44572b8fc2939f6a6f797aa496b58 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 15:24:28 +0000
Subject: [PATCH 372/395] Expand build/test.

---
 ci-overview.md | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/ci-overview.md b/ci-overview.md
index 47f71ea9dd5..a47cb7a2ff7 100644
--- a/ci-overview.md
+++ b/ci-overview.md
@@ -10,7 +10,9 @@ We use GitHub Actions for this purpose. It is flexible, powerful, and well-integ
 
 This document will walk you through our CI workflow and guide you on how to interact with and troubleshoot it when needed.
 
-## TL;DR - Workflow Overview
+## Workflow Overview
+
+### TL;DR
 
 ```mermaid
 graph LR
@@ -31,12 +33,36 @@ This repository relies on a GitHub Actions-based Continuous Integration (CI) wor
 - **Failures:** If a job fails, you'll be notified through GitHub's interface. You can then check the logs for details.
 - **Recovery:** To handle job failures, pull the relevant container image and rerun the script locally to reproduce the issue.
 
-## The Matrix
+### The Matrix
 
 The matrix defined in the [`matrix.yml`](ci/matrix.yaml) is the single source of truth for the environments we test our code against.
 It dictates the build configurations, such as CUDA version, operating system, CPU architecture, compiler, GPU architectures, and C++ standards. 
 It allows us to test our code against different combinations of these variables to ensure our code's compatibility and stability. 
 
+### Build and Test Jobs
+Our CI workflow primarily revolves around two major types of jobs: build jobs and test jobs. 
+
+#### Build
+Build jobs compile our unit tests and examples in various environments, mimicking the conditions in which our users might compile the code. 
+These jobs simply invoke a build script (e.g., `build_thrust.sh`) which contains all the necessary steps to compile the code.
+
+The advantage of this approach is two-fold. 
+First, it allows us to keep our CI configuration files clean and focused on the orchestration of jobs rather than the specifics of building the code. 
+Second, it greatly simplifies the process of reproducing build issues outside of CI. 
+Developers can run the build script locally, in their environment, and the build will behave in the same way it does in CI.
+
+#### Test
+After the build jobs have successfully compiled the test binaries, the test jobs run these binaries to execute our tests. 
+On first glance, you may notice that the test jobs are rebuilding all of the test binaries. 
+However, we are relying on sccache to cache the build artifacts from the build jobs and reuse them in the test jobs.
+
+Similar to the build jobs, test jobs use a script (e.g., `test_thrust.sh`) to define the steps required to execute the tests.
+If a test fails in CI, developers can simply run the script in their local environment to reproduce the issue and debug it.
+
+In summary, the heart of our build and test jobs is the corresponding build or test script. 
+This design philosophy helps maintain a clear separation between CI orchestration and the specifics of building and testing. 
+Moreover, it paves the way for straightforward issue reproduction, thereby aiding developers in their debugging process.
+
 ## Lifecycle of a Pull Request
 
 From creation to merging, a pull request in this project follows these steps:

From e603fb1c1df45d8aab526f6ce70207103de3369b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 15:36:01 +0000
Subject: [PATCH 373/395] Add examples.

---
 ci-overview.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ci-overview.md b/ci-overview.md
index a47cb7a2ff7..d511f5694e9 100644
--- a/ci-overview.md
+++ b/ci-overview.md
@@ -59,6 +59,15 @@ However, we are relying on sccache to cache the build artifacts from the build j
 Similar to the build jobs, test jobs use a script (e.g., `test_thrust.sh`) to define the steps required to execute the tests.
 If a test fails in CI, developers can simply run the script in their local environment to reproduce the issue and debug it.
 
+The syntax of the build and test scripts is the same:
+```bash
+./ci/build_thrust.sh <host compiler> <c++ standard> <gpu architectures>
+./ci/test_thrust.sh <host compiler> <c++ standard> <gpu architectures>
+
+#examples
+./ci/build_thrust.sh g++ c++17 70;80;86
+```
+
 In summary, the heart of our build and test jobs is the corresponding build or test script. 
 This design philosophy helps maintain a clear separation between CI orchestration and the specifics of building and testing. 
 Moreover, it paves the way for straightforward issue reproduction, thereby aiding developers in their debugging process.

From dc762e532ec40e1911266ae49e5b1e70523ed21a Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 15:53:14 +0000
Subject: [PATCH 374/395] Echo command to clone PR on failure.

---
 .github/workflows/build-and-test.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 562c7e7b7d3..2d1afc4c395 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -54,6 +54,11 @@ jobs:
         run: |
           hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt  2>&1 >/dev/null)
           echo "sccache hit rate: $hit_rate" >> $GITHUB_STEP_SUMMARY
+      - name: Provide instructions to reproduce this build
+        if: ${{ failure() }}
+        run: |
+          echo "To reproduce this build locally, run the following commands:" >> $GITHUB_STEP_SUMMARY
+          echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}

From bed0bff4b692de1a26ebf5cc22e8b9b1fb8d5f45 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 17:42:45 +0000
Subject: [PATCH 375/395] Factor image name into environment variable.

---
 .github/workflows/build-and-test.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 2d1afc4c395..eb2bbbc2675 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -23,9 +23,11 @@ jobs:
     if: inputs.build_script != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
+    env:
+      IMAGE_NAME: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
     container: 
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
+      image: ${{ env.IMAGE_NAME }}
     permissions:
       id-token: write
     steps:
@@ -57,7 +59,7 @@ jobs:
       - name: Provide instructions to reproduce this build
         if: ${{ failure() }}
         run: |
-          echo "To reproduce this build locally, run the following commands:" >> $GITHUB_STEP_SUMMARY
+          echo "::notice To reproduce this build locally, run the following commands:" 
           echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build

From 121e8b1a96412daef81d8b8c39296aa9fc96a692 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 17:55:40 +0000
Subject: [PATCH 376/395] Pass build and test job images as inputs.

---
 .github/workflows/build-and-test.yml          | 12 ++++++------
 .github/workflows/dispatch-build-and-test.yml |  2 ++
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index eb2bbbc2675..652cfc4eb91 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -17,17 +17,17 @@ on:
       os: {type: string, required: true}
       build_script: {type: string, required: false}
       test_script: {type: string, required: false }
+      build_image: {type: string, required: false}
+      test_image: {type: string, required: false}
 
 jobs:
   build:
-    if: inputs.build_script != ''
+    if: inputs.build_script != '' && inputs.build_image != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
-    env:
-      IMAGE_NAME: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
     container: 
       options: -u root
-      image: ${{ env.IMAGE_NAME }}
+      image: ${{ inputs.build_image }}
     permissions:
       id-token: write
     steps:
@@ -63,12 +63,12 @@ jobs:
           echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build
-    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' }}
+    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' && inputs.test_image != '' }}
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
     container: 
       options: -u root
-      image: rapidsai/devcontainers:23.06-cpp-${{inputs.compiler}}${{inputs.compiler_version}}-cuda${{inputs.cuda_version}}-${{inputs.os}}
+      image: ${{ inputs.test_image }}
       env:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
     permissions:
diff --git a/.github/workflows/dispatch-build-and-test.yml b/.github/workflows/dispatch-build-and-test.yml
index c5f1e955fe8..39e4daf5533 100644
--- a/.github/workflows/dispatch-build-and-test.yml
+++ b/.github/workflows/dispatch-build-and-test.yml
@@ -28,5 +28,7 @@ jobs:
       cpu: ${{ matrix.cpu }}
       os: ${{ matrix.os }}
       build_script: ${{ inputs.build_script }}
+      build_image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}}
       test_script: ${{ inputs.test_script }}
+      test_image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}}
 

From f025df29824e104ab736654f2005aeb5f167db79 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 18:10:51 +0000
Subject: [PATCH 377/395] Fix notice syntax.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 652cfc4eb91..d405a59553c 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Provide instructions to reproduce this build
         if: ${{ failure() }}
         run: |
-          echo "::notice To reproduce this build locally, run the following commands:" 
+          echo "::notice::To reproduce this build locally, run the following commands:" 
           echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY
   test:
     needs: build

From adeebed80d1a1e7bc2a1c538db0d1ad64480399c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 18:11:07 +0000
Subject: [PATCH 378/395] Update CUDA11.0 to 11.1 in matrix.

---
 ci/matrix.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
index 03f7321f08d..7a78a9ea183 100644
--- a/ci/matrix.yaml
+++ b/ci/matrix.yaml
@@ -22,16 +22,16 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
 - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '11.0', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
 - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}

From e6f34a6554b4e7826d2dfceff65d1ed8c123e878 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 18:36:06 +0000
Subject: [PATCH 379/395] Explicitly check exit code from script.

---
 .github/workflows/build-and-test.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d405a59553c..d60167802cd 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,7 +50,11 @@ jobs:
       - name: Run build script
         run: | 
             sccache -s > sccache_before.txt
-            time ./${{ inputs.build_script }} "${{inputs.compiler_exe}}" "${{inputs.std}}" "${{inputs.gpu_build_archs}}"
+            cmd="./${{ inputs.build_script }} \"${{inputs.compiler_exe}}\" \"${{inputs.std}}\" \"${{inputs.gpu_build_archs}}\""
+            eval $cmd || exit_code=$?
+            if [ ! -z "$exit_code" ]; then
+                echo "::error::Command '$cmd' failed with exit code $exit_code"
+            fi
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate
         run: |
@@ -60,7 +64,7 @@ jobs:
         if: ${{ failure() }}
         run: |
           echo "::notice::To reproduce this build locally, run the following commands:" 
-          echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY
+          echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' && inputs.test_image != '' }}

From 0b4a73330c47588820fb8cf6f810940b36f49e2f Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 18:46:31 +0000
Subject: [PATCH 380/395] Update to exit with the exit code on error.

---
 .github/workflows/build-and-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index d60167802cd..b78065d8722 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -54,6 +54,7 @@ jobs:
             eval $cmd || exit_code=$?
             if [ ! -z "$exit_code" ]; then
                 echo "::error::Command '$cmd' failed with exit code $exit_code"
+                exit $exit_code
             fi
             sccache -s > sccache_after.txt
       - name: Calculate sccache hit rate

From ed32d99d6a9e0e81d63fa7b024a153ab8a960296 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 19:16:34 +0000
Subject: [PATCH 381/395] Add repro instructions.

---
 .github/workflows/build-and-test.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b78065d8722..b3f4200785a 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -54,6 +54,9 @@ jobs:
             eval $cmd || exit_code=$?
             if [ ! -z "$exit_code" ]; then
                 echo "::error::Command '$cmd' failed with exit code $exit_code"
+                echo "To checkout the corresponding code and reproduce this build locally, run the following commands:" 
+                echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
+                echo "docker run --rm -it --gpus all --pull=always --network=host --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
                 exit $exit_code
             fi
             sccache -s > sccache_after.txt
@@ -61,11 +64,6 @@ jobs:
         run: |
           hit_rate=$(./ci/sccache_hit_rate.sh sccache_before.txt sccache_after.txt  2>&1 >/dev/null)
           echo "sccache hit rate: $hit_rate" >> $GITHUB_STEP_SUMMARY
-      - name: Provide instructions to reproduce this build
-        if: ${{ failure() }}
-        run: |
-          echo "::notice::To reproduce this build locally, run the following commands:" 
-          echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
   test:
     needs: build
     if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' && inputs.test_image != '' }}

From 7478f9c93f5c4833da4ae926ffe2a8c0803ccbe2 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 19:42:44 +0000
Subject: [PATCH 382/395] Update repro instructions.

---
 .github/workflows/build-and-test.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b3f4200785a..52d3d99f21e 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -50,12 +50,11 @@ jobs:
       - name: Run build script
         run: | 
             sccache -s > sccache_before.txt
-            cmd="./${{ inputs.build_script }} \"${{inputs.compiler_exe}}\" \"${{inputs.std}}\" \"${{inputs.gpu_build_archs}}\""
+            cmd="${{ inputs.build_script }} \"${{inputs.compiler_exe}}\" \"${{inputs.std}}\" \"${{inputs.gpu_build_archs}}\""
             eval $cmd || exit_code=$?
             if [ ! -z "$exit_code" ]; then
-                echo "::error::Command '$cmd' failed with exit code $exit_code"
-                echo "To checkout the corresponding code and reproduce this build locally, run the following commands:" 
-                echo "git clone --branch $(echo $GITHUB_REF | cut -d'/' -f3) --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
+                echo "::error::Build failed! To checkout the corresponding code and reproduce this build locally, run the following commands:" 
+                echo "git clone --branch $GITHUB_REF_NAME --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
                 echo "docker run --rm -it --gpus all --pull=always --network=host --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
                 exit $exit_code
             fi

From f12dacae60a2026f39a6d61bbf00f5098cb98bac Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 19:53:16 +0000
Subject: [PATCH 383/395] Update clone instruction to recurse submodules.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 52d3d99f21e..7cb3055fb33 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -54,7 +54,7 @@ jobs:
             eval $cmd || exit_code=$?
             if [ ! -z "$exit_code" ]; then
                 echo "::error::Build failed! To checkout the corresponding code and reproduce this build locally, run the following commands:" 
-                echo "git clone --branch $GITHUB_REF_NAME --single-branch https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
+                echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
                 echo "docker run --rm -it --gpus all --pull=always --network=host --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
                 exit $exit_code
             fi

From eb3778f83869fe6a70c9c433bfba5acca3f31b7e Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 21:47:00 +0000
Subject: [PATCH 384/395] Don't use host networking.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index 7cb3055fb33..bda70a70e26 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -55,7 +55,7 @@ jobs:
             if [ ! -z "$exit_code" ]; then
                 echo "::error::Build failed! To checkout the corresponding code and reproduce this build locally, run the following commands:" 
                 echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
-                echo "docker run --rm -it --gpus all --pull=always --network=host --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
+                echo "docker run --rm -it --gpus all --pull=always --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
                 exit $exit_code
             fi
             sccache -s > sccache_after.txt

From fe1b66c0ae074caaef502da80ece0b54f0f9f58c Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Wed, 21 Jun 2023 22:17:21 +0000
Subject: [PATCH 385/395] Disable test jobs for now.

---
 .github/workflows/pr.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 3b7fd74c0ad..f61c9ff8341 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -36,7 +36,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
-      test_script: "./ci/test_thrust.sh"
+      test_script: #"./ci/test_thrust.sh"
 
   cub:
     name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -50,7 +50,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_cub.sh"
-      test_script: "./ci/test_cub.sh"
+      test_script: #"./ci/test_cub.sh"
   
   libcudacxx:
     name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -63,7 +63,7 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      test_script: "./ci/test_libcudacxx.sh" 
+      test_script: #"./ci/test_libcudacxx.sh" 
 
   # This job is the final job that runs after all other jobs and is used for branch protection status checks.
   # See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks

From 65c328c335fedb8bf0445221ea529d0fdb23a566 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 01:32:53 +0000
Subject: [PATCH 386/395] Update devcontainers.

---
 .devcontainer/cuda11.0-gcc7/devcontainer.json | 22 ----------
 .devcontainer/cuda11.0-gcc8/devcontainer.json | 22 ----------
 .devcontainer/cuda11.0-gcc9/devcontainer.json | 22 ----------
 .../cuda11.0-llvm9/devcontainer.json          | 22 ----------
 .devcontainer/cuda11.1-gcc7/devcontainer.json | 26 ++++++++++++
 .devcontainer/cuda11.1-gcc8/devcontainer.json | 26 ++++++++++++
 .devcontainer/cuda11.1-gcc9/devcontainer.json | 26 ++++++++++++
 .../cuda11.1-llvm9/devcontainer.json          | 26 ++++++++++++
 .../cuda12.1-gcc10/devcontainer.json          |  8 +++-
 .../cuda12.1-gcc11/devcontainer.json          |  8 +++-
 .../cuda12.1-gcc12/devcontainer.json          |  8 +++-
 .devcontainer/cuda12.1-gcc7/devcontainer.json |  8 +++-
 .devcontainer/cuda12.1-gcc8/devcontainer.json |  8 +++-
 .devcontainer/cuda12.1-gcc9/devcontainer.json |  8 +++-
 .../cuda12.1-llvm10/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm11/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm12/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm13/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm14/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm15/devcontainer.json         |  8 +++-
 .../cuda12.1-llvm16/devcontainer.json         | 21 ----------
 .../cuda12.1-llvm9/devcontainer.json          |  8 +++-
 .devcontainer/devcontainer.json               | 41 +++++++++++--------
 .devcontainer/make_devcontainers.sh           |  6 +--
 24 files changed, 208 insertions(+), 156 deletions(-)
 delete mode 100644 .devcontainer/cuda11.0-gcc7/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-gcc8/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-gcc9/devcontainer.json
 delete mode 100644 .devcontainer/cuda11.0-llvm9/devcontainer.json
 create mode 100644 .devcontainer/cuda11.1-gcc7/devcontainer.json
 create mode 100644 .devcontainer/cuda11.1-gcc8/devcontainer.json
 create mode 100644 .devcontainer/cuda11.1-gcc9/devcontainer.json
 create mode 100644 .devcontainer/cuda11.1-llvm9/devcontainer.json
 delete mode 100644 .devcontainer/cuda12.1-llvm16/devcontainer.json

diff --git a/.devcontainer/cuda11.0-gcc7/devcontainer.json b/.devcontainer/cuda11.0-gcc7/devcontainer.json
deleted file mode 100644
index 976ad3d8666..00000000000
--- a/.devcontainer/cuda11.0-gcc7/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc7"
-}
diff --git a/.devcontainer/cuda11.0-gcc8/devcontainer.json b/.devcontainer/cuda11.0-gcc8/devcontainer.json
deleted file mode 100644
index ccf2707b6c6..00000000000
--- a/.devcontainer/cuda11.0-gcc8/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc8"
-}
diff --git a/.devcontainer/cuda11.0-gcc9/devcontainer.json b/.devcontainer/cuda11.0-gcc9/devcontainer.json
deleted file mode 100644
index 8063d298f00..00000000000
--- a/.devcontainer/cuda11.0-gcc9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-gcc9"
-}
diff --git a/.devcontainer/cuda11.0-llvm9/devcontainer.json b/.devcontainer/cuda11.0-llvm9/devcontainer.json
deleted file mode 100644
index 52946df942c..00000000000
--- a/.devcontainer/cuda11.0-llvm9/devcontainer.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.0-ubuntu18.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ],
-  "name": "cuda11.0-llvm9"
-}
diff --git a/.devcontainer/cuda11.1-gcc7/devcontainer.json b/.devcontainer/cuda11.1-gcc7/devcontainer.json
new file mode 100644
index 00000000000..f62e61a40f6
--- /dev/null
+++ b/.devcontainer/cuda11.1-gcc7/devcontainer.json
@@ -0,0 +1,26 @@
+{
+  "shutdownAction": "stopContainer",
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda11.1-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.1-gcc7"
+}
diff --git a/.devcontainer/cuda11.1-gcc8/devcontainer.json b/.devcontainer/cuda11.1-gcc8/devcontainer.json
new file mode 100644
index 00000000000..9c7de5681e8
--- /dev/null
+++ b/.devcontainer/cuda11.1-gcc8/devcontainer.json
@@ -0,0 +1,26 @@
+{
+  "shutdownAction": "stopContainer",
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda11.1-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.1-gcc8"
+}
diff --git a/.devcontainer/cuda11.1-gcc9/devcontainer.json b/.devcontainer/cuda11.1-gcc9/devcontainer.json
new file mode 100644
index 00000000000..ecad339b940
--- /dev/null
+++ b/.devcontainer/cuda11.1-gcc9/devcontainer.json
@@ -0,0 +1,26 @@
+{
+  "shutdownAction": "stopContainer",
+  "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda11.1-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.1-gcc9"
+}
diff --git a/.devcontainer/cuda11.1-llvm9/devcontainer.json b/.devcontainer/cuda11.1-llvm9/devcontainer.json
new file mode 100644
index 00000000000..56bb6c44e70
--- /dev/null
+++ b/.devcontainer/cuda11.1-llvm9/devcontainer.json
@@ -0,0 +1,26 @@
+{
+  "shutdownAction": "stopContainer",
+  "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda11.1-ubuntu18.04",
+  "hostRequirements": {
+    "gpu": true
+  },
+  "initializeCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -p .cache/.{aws,config}"
+  ],
+  "containerEnv": {
+    "SCCACHE_REGION": "us-east-2",
+    "SCCACHE_BUCKET": "rapids-sccache-devs",
+    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+  },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+  ],
+  "name": "cuda11.1-llvm9"
+}
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
index c8d8d19c4f3..78ab69a47af 100644
--- a/.devcontainer/cuda12.1-gcc10/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc10-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc10"
 }
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
index 5d31a63dfc0..da3fe0c2356 100644
--- a/.devcontainer/cuda12.1-gcc11/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc11-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc11"
 }
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
index f66384fc7d8..6d98558fca8 100644
--- a/.devcontainer/cuda12.1-gcc12/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc12"
 }
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
index b6b6f982949..eec4f8ddad7 100644
--- a/.devcontainer/cuda12.1-gcc7/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc7-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc7"
 }
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
index 6fffb8aff1f..f3516109956 100644
--- a/.devcontainer/cuda12.1-gcc8/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc8-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc8"
 }
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
index a68bc776134..4ab911547fe 100644
--- a/.devcontainer/cuda12.1-gcc9/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-gcc9-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc9"
 }
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
index 10307b85907..fd494ba5cf5 100644
--- a/.devcontainer/cuda12.1-llvm10/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm10-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm10"
 }
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
index 5bda646de84..5e38f7851c8 100644
--- a/.devcontainer/cuda12.1-llvm11/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm11-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm11"
 }
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
index 748aec1361d..f5506695cc9 100644
--- a/.devcontainer/cuda12.1-llvm12/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm12-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm12"
 }
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
index 2b4652a454d..f2d515a06c3 100644
--- a/.devcontainer/cuda12.1-llvm13/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm13-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm13"
 }
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
index eb4eee24b2b..82fc4c85060 100644
--- a/.devcontainer/cuda12.1-llvm14/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm14-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm14"
 }
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
index 8bbcfaf933a..7ec66036de6 100644
--- a/.devcontainer/cuda12.1-llvm15/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm15-cuda12.1-ubuntu22.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm15"
 }
diff --git a/.devcontainer/cuda12.1-llvm16/devcontainer.json b/.devcontainer/cuda12.1-llvm16/devcontainer.json
deleted file mode 100644
index d020ee58aa3..00000000000
--- a/.devcontainer/cuda12.1-llvm16/devcontainer.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "image": "rapidsai/devcontainers:23.06-cpp-llvm16-cuda12.1-ubuntu22.04",
-  "hostRequirements": {
-    "gpu": true
-  },
-  "initializeCommand": [
-    "/bin/bash",
-    "-c",
-    "mkdir -p .cache/.{aws,config}"
-  ],
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
-}
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
index d971fff7795..92bf736e022 100644
--- a/.devcontainer/cuda12.1-llvm9/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -1,4 +1,5 @@
 {
+  "shutdownAction": "stopContainer",
   "image": "rapidsai/devcontainers:23.06-cpp-llvm9-cuda12.1-ubuntu20.04",
   "hostRequirements": {
     "gpu": true
@@ -14,9 +15,12 @@
     "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
     "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
   },
+  "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm9"
 }
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index b60dd7cfc78..0266f542962 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,20 +1,25 @@
-
 {
-  "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
-
-  "hostRequirements": { "gpu": true },
-
-  "initializeCommand": ["/bin/bash", "-c", "mkdir -p .cache/.{aws,config}"],
-
-  "containerEnv": {
-    "SCCACHE_REGION": "us-east-2",
-    "SCCACHE_BUCKET": "rapids-sccache-devs",
-    "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
-    "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
-  },
-
-  "mounts": [
-    "source=${localWorkspaceFolder}/.cache/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/.cache/.config,target=/home/coder/.config,type=bind,consistency=consistent"
-  ]
+    "shutdownAction": "stopContainer",
+    "image": "rapidsai/devcontainers:23.06-cpp-gcc12-cuda12.1-ubuntu22.04",
+    "hostRequirements": {
+        "gpu": true
+    },
+    "initializeCommand": [
+        "/bin/bash",
+        "-c",
+        "mkdir -p .cache/.{aws,config}"
+    ],
+    "containerEnv": {
+        "SCCACHE_REGION": "us-east-2",
+        "SCCACHE_BUCKET": "rapids-sccache-devs",
+        "VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
+        "HISTFILE": "${containerWorkspaceFolder}/.cache/._bash_history"
+    },
+    "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+    "mounts": [
+        "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+        "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+        "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    ]
 }
\ No newline at end of file
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
index 7e5e430cfc5..f6fe40d1dcf 100755
--- a/.devcontainer/make_devcontainers.sh
+++ b/.devcontainer/make_devcontainers.sh
@@ -20,19 +20,19 @@ IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"
 matrix_json=$(yq -o json ../ci/matrix.yaml)
 
 # Get unique combinations of cuda version, compiler name/version, and Ubuntu version
-combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, ubuntu: .ubuntu}] | unique | .[]')
+combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, os: .os}] | unique | .[]')
 
 # For each unique combination
 for combination in $combinations; do
     cuda_version=$(echo "$combination" | jq -r '.cuda')
     compiler_name=$(echo "$combination" | jq -r '.compiler_name')
     compiler_version=$(echo "$combination" | jq -r '.compiler_version')
-    ubuntu_version=$(echo "$combination" | jq -r '.ubuntu')
+    os=$(echo "$combination" | jq -r '.os')
 
     name="cuda$cuda_version-$compiler_name$compiler_version"
     mkdir -p "$name"
     devcontainer_file="$name/devcontainer.json"
-    image="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-ubuntu$ubuntu_version"
+    image="$IMAGE_ROOT$compiler_name$compiler_version-cuda$cuda_version-$os"
 
     # Use the base_devcontainer.json as a template, plug in the CUDA, compiler names, versions, and Ubuntu version,
     # and write the output to the new devcontainer.json file

From 69f45d6c031841ca459bc786fc817cd4791e863b Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 02:35:09 +0000
Subject: [PATCH 387/395] Update devcontainers.

---
 .devcontainer/cuda11.1-gcc7/devcontainer.json   | 8 ++++----
 .devcontainer/cuda11.1-gcc8/devcontainer.json   | 8 ++++----
 .devcontainer/cuda11.1-gcc9/devcontainer.json   | 8 ++++----
 .devcontainer/cuda11.1-llvm9/devcontainer.json  | 8 ++++----
 .devcontainer/cuda12.1-gcc10/devcontainer.json  | 8 ++++----
 .devcontainer/cuda12.1-gcc11/devcontainer.json  | 8 ++++----
 .devcontainer/cuda12.1-gcc12/devcontainer.json  | 8 ++++----
 .devcontainer/cuda12.1-gcc7/devcontainer.json   | 8 ++++----
 .devcontainer/cuda12.1-gcc8/devcontainer.json   | 8 ++++----
 .devcontainer/cuda12.1-gcc9/devcontainer.json   | 8 ++++----
 .devcontainer/cuda12.1-llvm10/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm11/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm12/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm13/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm14/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm15/devcontainer.json | 8 ++++----
 .devcontainer/cuda12.1-llvm9/devcontainer.json  | 8 ++++----
 .devcontainer/devcontainer.json                 | 8 ++++----
 .gitignore                                      | 4 +++-
 19 files changed, 75 insertions(+), 73 deletions(-)

diff --git a/.devcontainer/cuda11.1-gcc7/devcontainer.json b/.devcontainer/cuda11.1-gcc7/devcontainer.json
index f62e61a40f6..906f4d90301 100644
--- a/.devcontainer/cuda11.1-gcc7/devcontainer.json
+++ b/.devcontainer/cuda11.1-gcc7/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda11.1-gcc7"
 }
diff --git a/.devcontainer/cuda11.1-gcc8/devcontainer.json b/.devcontainer/cuda11.1-gcc8/devcontainer.json
index 9c7de5681e8..c259ad58fa1 100644
--- a/.devcontainer/cuda11.1-gcc8/devcontainer.json
+++ b/.devcontainer/cuda11.1-gcc8/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda11.1-gcc8"
 }
diff --git a/.devcontainer/cuda11.1-gcc9/devcontainer.json b/.devcontainer/cuda11.1-gcc9/devcontainer.json
index ecad339b940..096ed56752e 100644
--- a/.devcontainer/cuda11.1-gcc9/devcontainer.json
+++ b/.devcontainer/cuda11.1-gcc9/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda11.1-gcc9"
 }
diff --git a/.devcontainer/cuda11.1-llvm9/devcontainer.json b/.devcontainer/cuda11.1-llvm9/devcontainer.json
index 56bb6c44e70..dd64ac5f050 100644
--- a/.devcontainer/cuda11.1-llvm9/devcontainer.json
+++ b/.devcontainer/cuda11.1-llvm9/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda11.1-llvm9"
 }
diff --git a/.devcontainer/cuda12.1-gcc10/devcontainer.json b/.devcontainer/cuda12.1-gcc10/devcontainer.json
index 78ab69a47af..9313ddce140 100644
--- a/.devcontainer/cuda12.1-gcc10/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc10/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc10"
 }
diff --git a/.devcontainer/cuda12.1-gcc11/devcontainer.json b/.devcontainer/cuda12.1-gcc11/devcontainer.json
index da3fe0c2356..73dad29d001 100644
--- a/.devcontainer/cuda12.1-gcc11/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc11/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc11"
 }
diff --git a/.devcontainer/cuda12.1-gcc12/devcontainer.json b/.devcontainer/cuda12.1-gcc12/devcontainer.json
index 6d98558fca8..62772435f02 100644
--- a/.devcontainer/cuda12.1-gcc12/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc12/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc12"
 }
diff --git a/.devcontainer/cuda12.1-gcc7/devcontainer.json b/.devcontainer/cuda12.1-gcc7/devcontainer.json
index eec4f8ddad7..e4eb3ccfbdf 100644
--- a/.devcontainer/cuda12.1-gcc7/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc7/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc7"
 }
diff --git a/.devcontainer/cuda12.1-gcc8/devcontainer.json b/.devcontainer/cuda12.1-gcc8/devcontainer.json
index f3516109956..8cf373bc889 100644
--- a/.devcontainer/cuda12.1-gcc8/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc8/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc8"
 }
diff --git a/.devcontainer/cuda12.1-gcc9/devcontainer.json b/.devcontainer/cuda12.1-gcc9/devcontainer.json
index 4ab911547fe..513b9206bfe 100644
--- a/.devcontainer/cuda12.1-gcc9/devcontainer.json
+++ b/.devcontainer/cuda12.1-gcc9/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-gcc9"
 }
diff --git a/.devcontainer/cuda12.1-llvm10/devcontainer.json b/.devcontainer/cuda12.1-llvm10/devcontainer.json
index fd494ba5cf5..3f78e5990df 100644
--- a/.devcontainer/cuda12.1-llvm10/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm10/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm10"
 }
diff --git a/.devcontainer/cuda12.1-llvm11/devcontainer.json b/.devcontainer/cuda12.1-llvm11/devcontainer.json
index 5e38f7851c8..a465e8c4268 100644
--- a/.devcontainer/cuda12.1-llvm11/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm11/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm11"
 }
diff --git a/.devcontainer/cuda12.1-llvm12/devcontainer.json b/.devcontainer/cuda12.1-llvm12/devcontainer.json
index f5506695cc9..a6d3372480a 100644
--- a/.devcontainer/cuda12.1-llvm12/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm12/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm12"
 }
diff --git a/.devcontainer/cuda12.1-llvm13/devcontainer.json b/.devcontainer/cuda12.1-llvm13/devcontainer.json
index f2d515a06c3..133229967c6 100644
--- a/.devcontainer/cuda12.1-llvm13/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm13/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm13"
 }
diff --git a/.devcontainer/cuda12.1-llvm14/devcontainer.json b/.devcontainer/cuda12.1-llvm14/devcontainer.json
index 82fc4c85060..0a290a0dc49 100644
--- a/.devcontainer/cuda12.1-llvm14/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm14/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm14"
 }
diff --git a/.devcontainer/cuda12.1-llvm15/devcontainer.json b/.devcontainer/cuda12.1-llvm15/devcontainer.json
index 7ec66036de6..c2729aaaebf 100644
--- a/.devcontainer/cuda12.1-llvm15/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm15/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm15"
 }
diff --git a/.devcontainer/cuda12.1-llvm9/devcontainer.json b/.devcontainer/cuda12.1-llvm9/devcontainer.json
index 92bf736e022..44297c6183f 100644
--- a/.devcontainer/cuda12.1-llvm9/devcontainer.json
+++ b/.devcontainer/cuda12.1-llvm9/devcontainer.json
@@ -7,7 +7,7 @@
   "initializeCommand": [
     "/bin/bash",
     "-c",
-    "mkdir -p .cache/.{aws,config}"
+    "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
   ],
   "containerEnv": {
     "SCCACHE_REGION": "us-east-2",
@@ -18,9 +18,9 @@
   "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
   "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
   "mounts": [
-    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
   ],
   "name": "cuda12.1-llvm9"
 }
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 0266f542962..0a86dc0c7d3 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -7,7 +7,7 @@
     "initializeCommand": [
         "/bin/bash",
         "-c",
-        "mkdir -p .cache/.{aws,config}"
+        "mkdir -m 0755 -p ${localWorkspaceFolder}/.{aws,cache,config}"
     ],
     "containerEnv": {
         "SCCACHE_REGION": "us-east-2",
@@ -18,8 +18,8 @@
     "workspaceFolder": "/home/coder/${localWorkspaceFolderBasename}",
     "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
     "mounts": [
-        "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
-        "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
-        "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent"
+        "source=${localWorkspaceFolder}/.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+        "source=${localWorkspaceFolder}/.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+        "source=${localWorkspaceFolder}/.config,target=/home/coder/.config,type=bind,consistency=consistent"
     ]
 }
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 2d7da49b38e..a221dae54e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
 .idea/
 *build*/
-.cache
\ No newline at end of file
+.cache
+.aws
+.config
\ No newline at end of file

From 33bde08b0e27b373ee0ed1395d9ac9fc02384a95 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 02:56:08 +0000
Subject: [PATCH 388/395] Add launch script and repro instructions.

---
 .devcontainer/launch.sh              | 45 ++++++++++++++++++++++++++++
 .github/workflows/build-and-test.yml |  5 ++++
 2 files changed, 50 insertions(+)
 create mode 100755 .devcontainer/launch.sh

diff --git a/.devcontainer/launch.sh b/.devcontainer/launch.sh
new file mode 100755
index 00000000000..984d6412382
--- /dev/null
+++ b/.devcontainer/launch.sh
@@ -0,0 +1,45 @@
+
+#! /usr/bin/env bash
+
+launch_devcontainer() {
+
+    # Ensure we're in the repo root
+    cd "$( cd "$( dirname "$(realpath -m "${BASH_SOURCE[0]}")" )" && pwd )/..";
+
+    if [[ -z $1 ]] || [[ -z $2 ]]; then
+        echo "Usage: $0 [CUDA version] [Host compiler]"
+        echo "Example: $0 12.1 gcc12"
+        return 1
+    fi
+
+    local cuda_version="$1"
+    local host_compiler="$2"
+    local workspace="$(basename "$(pwd)")";
+    local tmpdir="$(mktemp -d)/${workspace}";
+    local path="$(pwd)/.devcontainer/cuda${cuda_version}-${host_compiler}";
+
+    mkdir -p "${tmpdir}";
+    mkdir -p "${tmpdir}/.devcontainer";
+    cp -arL "$path/devcontainer.json" "${tmpdir}/.devcontainer";
+    sed -i "s@\${localWorkspaceFolder}@$(pwd)@g" "${tmpdir}/.devcontainer/devcontainer.json";
+    path="${tmpdir}";
+
+    local hash="$(echo -n "${path}" | xxd -pu - | tr -d '[:space:]')";
+    local url="vscode://vscode-remote/dev-container+${hash}/home/coder/cccl";
+
+    echo "devcontainer URL: ${url}";
+
+    local launch="";
+    if type open >/dev/null 2>&1; then
+        launch="open";
+    elif type xdg-open >/dev/null 2>&1; then
+        launch="xdg-open";
+    fi
+
+    if [ -n "${launch}" ]; then
+        code --new-window "${tmpdir}";
+        exec "${launch}" "${url}" >/dev/null 2>&1;
+    fi
+}
+
+launch_devcontainer "$@";
\ No newline at end of file
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index bda70a70e26..b55e50c9a65 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -56,6 +56,11 @@ jobs:
                 echo "::error::Build failed! To checkout the corresponding code and reproduce this build locally, run the following commands:" 
                 echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
                 echo "docker run --rm -it --gpus all --pull=always --volume \$PWD:/repo --workdir /repo ${{ inputs.build_image }} $cmd"
+                echo "Alternatively, for a more convenient, interactive environment to reproduce the issue, you can launch a devcontainer in vscode:"
+                echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
+                echo ".devcontainer/launch.sh ${{inputs.cuda_version}} ${{inputs.compiler}}${{inputs.compiler_version}}"
+                echo "Then, inside the devcontainer, run:"
+                echo "$cmd"
                 exit $exit_code
             fi
             sccache -s > sccache_after.txt

From 773c4a89fc9d392e7b993211625bf9a8d5da88dc Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 03:02:05 +0000
Subject: [PATCH 389/395] Update repro message.

---
 .github/workflows/build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index b55e50c9a65..a1c0c5ab121 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -59,7 +59,7 @@ jobs:
                 echo "Alternatively, for a more convenient, interactive environment to reproduce the issue, you can launch a devcontainer in vscode:"
                 echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
                 echo ".devcontainer/launch.sh ${{inputs.cuda_version}} ${{inputs.compiler}}${{inputs.compiler_version}}"
-                echo "Then, inside the devcontainer, run:"
+                echo "Then, open a terminal inside vscode (ctrl+shift+`) and run:"
                 echo "$cmd"
                 exit $exit_code
             fi

From bb785b13c69570432584ad6f63989f7210c1efd3 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 03:10:00 +0000
Subject: [PATCH 390/395] Add repro instructions.

---
 ci-overview.md                     |  14 ++++++++++++++
 docs/images/repro_instructions.png | Bin 0 -> 73132 bytes
 2 files changed, 14 insertions(+)
 create mode 100644 docs/images/repro_instructions.png

diff --git a/ci-overview.md b/ci-overview.md
index d511f5694e9..3334686c8dc 100644
--- a/ci-overview.md
+++ b/ci-overview.md
@@ -92,6 +92,20 @@ If a CI job fails, here's what you can do to troubleshoot:
 2. Reproduce the issue locally: Pull the relevant container image and rerun the script that failed. This will allow you to dig into the issue in depth.
 3. Fix the issue: Once you've identified the problem, you can make appropriate changes to your code and rerun the CI jobs.
 
+### How to Reproduce a CI Failure Locally
+
+When a build or test job fails, it will provide instructions on how to reproduce the failure locally using the exact same code and environment used in CI.
+
+For example, here is a screenshot of the log of a failed build job:
+
+![Build Job Failure](docs/images/repro_instructions.png)
+
+This provides instructions for both a command-line and a VSCode-based approach to reproduce the failure locally.
+
+When interating on a fix, the vscode devcontainer approach is recommended as it provides a convenient, interactive environment to debug the issue.
+
+## More Information
+
 You can refer to [GitHub Actions documentation](https://docs.github.com/en/actions) for a deeper understanding of the process and the [GitHub Actions workflows syntax](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions) to comprehend the workflow files' structure and syntax.
 
 You're not in this alone - the community is here to help. If you're stuck, don't hesitate to raise an issue or ask for assistance. Open source thrives on collaboration and learning. Happy coding!
diff --git a/docs/images/repro_instructions.png b/docs/images/repro_instructions.png
new file mode 100644
index 0000000000000000000000000000000000000000..f0579650823a244c34d737eead24dcdd4406053a
GIT binary patch
literal 73132
zcma&O1ymf}wm(<|Ngzmq2Tcg>8iEsoySux)TjK=yfDkOW1$TD|?h;%ZcX#ipy#KrR
zy}2`M%>aw0tBb1UoPGBG$v%Y1%Zj0)5TF16fF>a>tOx*zD&WuSuU>%Pe`8bxfDZ^}
zUu3=jKxOpnM*~Fg`FkUAMHv9_qyzxJAON@r5Bco^fD1DK?CS#n_fG)8vrlbO;00em
zGL#Y%24H_bGh6dx!6R>s3}jsa0ORkAf&c*L0^XqZf@<X@cojfG_=~dp!eN@5I@Z8L
z2b3LCoI1-*o+hB+(zI(uj?e3h$li2#tJluFml&59-@!Z`3T~5*eE21aBnrj+<FopJ
zLlq(X`m)c{$j;fBM96%S$50DfHCuC*`kXT&!V7=F_itYPp9cYU<zFxU_Yq(B@q5z$
zb;37N(wb+nf1m#fVZHBVDeUck|N7sD+cS%YE64+S0$=O>Cz;A)a~}QiasRI&JdI5g
z?@)Oal%$&Fpm2eqk@h!-qw<QKSZJuIURQgw%}^lPmT!(o_JBa&+Pcc~XaO2cyjba!
zn9|IX-fvh*W;zn}{A50r)wR(pcdN&_IAKIpQ&Y2PJU%;{!RK~dul$yTWvG9yK_n5C
z_!B+#XJKEGI+TaU;D~cVC|M&C#yd<Ft82)NVoJUB(R^{@VR2ZPgtGJHS!}+y>9Bcl
zg-v!B?nJ9sbnC-)2CvWd+*xdIFQxH3d&ANN9wPz4yC+`H%ciS?6f=8gXU$qqnH0{k
zMK4P&mmS}y2Qd+`41TZC>FJ|(B{iw#=0-d3+Oq!Oh~%c-`Z>D$egz*NACbf9y5;NQ
zvkIJo!r~)7`|qWl&t*?mPa!gn&rP;Zo{O?+ZC3D_Z+EtgEG<1(^|9YSLl>?3%9fvA
zqSjQ^@k7?Kt)oK0x%visVbl0*#*BZ&t*)-=W}R%VZ*FaFOmwljl9n>Nr`(hq&jF0@
zIh(1?sxc|q={A#Lh7AudzyYxX`}>=zr-uW9&GixCD9sBzwPo*|wnri&BGZnKTJ26&
zPQShKe!*$G<W3o}cXC4bj;ma*y)CWT`3_ft278cme}7TxdvsXXUVVbT<vAY#Wf2a8
zZbL;;(LVIY0jX)<wG9o;Kvj8TV`CdXKOU3rEv`YkLMVgbQV=qgEKvr#LCZ~?goj7t
zFH@XXXE4MU{_^jwkB_jZ<hUIk-52X3iwpPWTOX$B5*L5fG`k(&-ru(P++0JN%C_4+
zv@=%iUP9~DiWQizqu#My&qNZw$Xb1<71j{ZP|}J$EVs3__2!2irEo%32%$8iFtLAb
z1;7y?JQXlz%95wXL<xS8rA}{KP!7{`X8Rkc3R(3A8pr8;mbcU8x_RpA>O+_Z2UdD*
z;$_9fh<-o*7}_w{0OOOK1tqKT2?;<)L>xIUm6%GcEID{BKweeR(8Ojk{L{F3&Ciz@
z$4hN!fE~2W!rXlP&kW|~D^@nP6KFeBUQbU!_v{11^7#1a^iZO4@FTmUG3*(B*V$9j
z@lRF(b(ruAU`VaB>3JtZSGV=3$+@hs4EZ%OlHY_;A8Y(i7nj<ukf&~A){kVJl2Vd!
zDsetfj|>c@zwmLfE>1g1Apj6B$>+M>-Pzw?@$D5K-_y=m`q6yrYk*QVb#tPW5WwEt
zIazAc8FiklQ_Ani)TasQ>FpiF)S#b{))OGq(xMG^JX`PKskeg@yh-PK@fc|Ctf}nG
z4wPuN@p5t|N;AHE**CDVVp&v#gM;((`ebZ)81Rcs;Y>+ND%W`^7N(VukzA>@qSDY|
zCtzUy+&wy4)>6XD!z0z^l2EQ`Vrf}amPfLCqNb}GMMc8p`nVakAv?-=@9F6Vj+y5c
z%KVnG<HX)&<He4ahDKt1yp3jiy4pb3R`UAMh%w8M<WQa6ayS)cc1a2ENMGOY-wG&y
z8pNWbNd+k5CBdK^GwQuOoQM_5`UMe>=C;qQt8+bRdy-4%cW`rRci#J<cCykVBEnN=
z$HvB)!DIIv+b{=?^TENs>&Z%ddIA!FMJap8sSjucx4MlTi?OgcP6_2IK0U%WIqwPO
zPG<D>lJI#7M^yxIv$K9efAwl=)sb-jK#oWtJv9{ph<S}?YHF;hu5NB=fs2c43vI&b
zV(vtGgR)o-fg#6I78Vwbr0#M0^r8Z<&Ts6@&EL~!m9?RFk$YzeQe&^{G|()2Ix{ga
z$jHfADd??KSSVd)b)694l}SqHeiVzAOaJ_Zvgh^;>$$~Kg&fo8xbCAUDhY3pXpYbD
zZG+jYEq{9mea!gI)&<i{k3RD@le)f*Nm^oJz01XR(SFhIuOaFh8g|e+8~_QG$k@mz
z5JGv0Cd6LLZZ?*bKd@zLYC_AAK0sEj=F})HBa_Ieb$qfD@vst(Lw7a?tgi1bl<Pro
zaN?`Y#@0UsNHccYO5x|+W8i-_+!QB>PmYbPwpqlDk)O(Tp9`NW&M7`vUnh9`HUx5H
zJ~=VQ<kw?}92BtJ?CLW(iUgG9ZKpOm)w0+A`SS-@9Z8vWJ6ZaGZLnLhb~HR<;oOu^
zmw@@z;^`jhj)TM9)I|8n+R$mv{iT4<Ik|5mJ*9LiugAlT&8=D)5~ueJJ+Tfk@Y!YE
zSX-N(#49i$fX8J|QB(Wzd`rsc<A%**-J4Gx!%Mupyksmi6Ms4twY0oo&pvj#?UWH0
z#_Vc%Nu-i+y>dTSr}OQ;>MV~E^YD9%Q_<TmID@s${&`75V-U>r(ZqcB&&$o*XWz%a
z8CTYGg@}rN_>iSCt<u!wiE#`)nQezf9?aT7n<NStSwr7(ThCQwh@M&%*)2D4`F0`*
zK$8G{ef?EZ{DRWbk1^kF4p(Gm09InmUfrrY(TKz8mc<|QQe^Q_?|90<#DO1jw9t4#
zeAg<}S^1+xJw~4T&8r}?0G`hODynW4Lr<_imK&v5%md4VWXc3^p(Mt``GtpvhZTB!
z`N9u?MP+t+w%N<|gwy_8PBmR%76LFGA1^5>jfRE>Ff+5$GSJwH5RzjKYz-wcqvlMQ
z!@oqMWh(C<8iMzmo9nwh&PCD6%_$uTy~Hr4Xz+4J#b>sk=<NTVDF{y8_pGmHm)QXU
zd;I)Ny1zz$eq!Kd|HV>Sp;d2te~;u3_Z&+Z#YD$|=!c9&spyCK7PrZGBD>A!!DVj&
z2Vh2}qholD7V(YW**y6suGPyG-Y+RBskr%}<h0^+{CgskuKJ>L+(%7$`~EN^v5%cy
zv~(VWw1e*Jn=~uZlx)20#=isZ1fofn+k9RE-+V4_=PGlMk<Y*)1Ag5$#mF;)8Sxue
zwcGI=7{Fc^+mVeI+iO~A7dxY+Wrey;&XwiS(bhk{s3)4$+vZW`8GQKN{#e!Iv@?2M
zTJ70`qDM;Vea{Hr#jHcTtT$Vx(dxJb-dR%mPhdchBb?xD+4as?+Uv`k;?1*r3?krm
z%6p}S&&xF!=_OS@`AY=E0gGwye0Bmp>JEsjdqZoEQMs-zxPI0ba>SxVB5;SpUzxFJ
z<cLNi0kn|`&(BZG&2BB8H@bDkl_|}HX|joiKhx7mqWfN>oPsGU8zT5#_i#@s-`mGq
zQBkpNzoyLgc!roKBs6$5jd-HEJ~1~pp2#5O;JZ-Pa5~$Lg0!@|d4E|NkJ(6i*IK~~
zfbw_K!lK<mgMopu@xwLoA@_U?ho8K<H&lHa2^=uv2uP}=uE|%QR(a~jbrRf2Q9=RV
z^??m4&4t9*=O~wGmvDk#bpj=G3Gnd~u<?d)VPNFDrXATXw@l30@OgP$r3={U>ia=t
zTt;g4CI>JFymirGhC0CN>aP9u@H=P{8?8sYK=ixiKhGll4QE(YtZYI7furgCHa9o6
z55cXkwtfUAHQpU+yroksJ3ULvD$8ROE8AX9U>3C~up0i!<+N30^5ORvY+94mj^7NC
zp{3tiv8ZI5?AL^RmUm=^{5n=6@R))SUh?^z8;955m=Q-2W@KcL5@EYvY`bu}xqbhE
zPKr<I;OfNZdf0!SD<$A_&u+f@9-wBR(P(mZaFO9yX-sB(hRbw2;h&$M*U(71odwG+
zE)EVjr6+TBeAMJYlDRMZLi5d<(TT}Ag+AFVtWGaVp(s`%iL(rMPqFhc*jyFG?~G-n
zFgsxC(i@~weX^1Yp>=V&8XO#a4{0w}D@Q;;{9!UQHrB=SIdsH;mIbb(W1LxlGn$zM
zlOTrv`qRr;ae9W*>*INyfsMlAV$!+#da<$e&1<rPh~h#nkE)x&ts&gujFZ(}>Uhbs
zA3Gw9;@{+6y+&TCwz_#ld<_DK2xoD8uYqmC@Wo=0W!0}btwlw&)Xb|3wN_YEGM0Pl
z5+Q=`JE8-TMa-X~LEH#qc+R?kdAOcT+pT_=)b;q{O9D`{#9dxqt+aT0z(yHp<1MYs
zUIJ+AA+t<+&NusP?sKo$E?iHbb<ayVIy<BJvKhpDp2O+OJSoQ>fA%gls<+8|`-bL&
zom?F}urI#rb@eeHUXN&%*=9iBzsC%xrKJT+e5Aoo>qERSVFK;QuL>{E;L+Wis;h^t
z4rYHT(&%+WM@G6hJO40|m6i3rIZmTD)6pSLEMD6_9yP2i(`!#ijT4@sEkjaMQ{(*b
z0S+MHcYm)X^0~QbrO5@Wt0H+nxnZskM1BlEhE`TS@G+o4M?JBzu(<5frP%{8zwKAm
z*2;)~!;9!4%i6uW*eR`6)7B;e;^Gs-gvrp*Y^9hW8P~^3u64?^J{R}3W$7fq+~T5{
ziK(eK2y(P$?dEtsGbh_P4%$1ow7Og%g&2^<nD4ENfas%h`fBwVK1=YM!Ca{pCowTG
zoZpM@PkTx-GBUopB;bizuOLmrWHGB^vgzHuNhv9GbaeFvPh;u)8gIDt$YNe0cFJkY
zdm?!<G-*pIN$EjXpP#&d@A2_Tv9U%*7k7u!j~8Z!mH1s1a`W?=+zXE0IgJ$z6lA=p
zfZ~r>4AT3>$zSkD=+a&{vZK}T#+!JX2M2_^2L~s7_8lY3AlzhT5|F!|m58Ox5O9;F
z$KBA=(a|C8={~*Vzz-1-A^wNk-jBXggdbKuiwL|UWjn<r>EquwH(I2Go1f{u-iyr#
zzNe(5#Ke5)(+zSQSo{6-fFH88v$eIorKzql7*p&~Pw|gJyh{9Jo-7R=Qy25$Q?uLc
z1EMAYh8DKh$nXdQJ^k_Wlhqe~$Z;x_g9C87JwoW?1J64#xWIjUW(0_PWhBJLeuUh*
z!>)BiJP-k6GqZucu+OLzxf4dcsr(*_368h&Q&|Op!1wfY(frA24)#+d@@|3h^5XKc
zqA19<wlDnt{{9b4EV%bPbW&Pk`MKyhxAKSpHVR^U+8OnBOrr-6ZiL9s()aCxdSd|K
zyV&5cf4OJ5Zwbh&=!oEAq!3n$RMux#W#=`8=yi>a4HXv^DJrSNb1rcf4-F2|e!MVp
zP<R_+zzQ9oo@D#<srSS(H$Oi<K4Ia<jl2-?j~?C5yO8yof<;5c!{f9?@K3y=Bj)qS
z!B|t5=p8*;6!3kN*V5vF925Pe*xeB1djX}9*g1U4*y%#4*-@P%y}Z0BPIhcO-03N0
zc){L2jHppYAhW2*)N1zbIZaE*zhk_AZlrp#%garZ<s7c#K&_neAJWCz;-_p8sRVir
zZEa-)035Kpi;ux%YhN0TK7@`q5UejX%jp4`@-&YG{OOE{7OJM(cuBd6777M$ef8x+
zkkoSW@{nNKIyyE61{1%6+S9%>(b6t78ueAIb$#vXR8&&DIe}{FsL0IGehU4FjdJGo
z4#W^uku>Nj#BW)o0t|qT%*m~i4&<%%f+CdL3^ywl)L2*roq-NfRFU;D-J4fa)6-f8
zOa?}+n9@QvcU=jrZ@V#(BPHPh^w7!Q*<CA78E86MYQ<%Fx~jUO?v1o!@lbQ7`88uS
zv`FyQ1OHGj4q$V#`(cw~vXrrSf0TWXy_#28TyAA<u3tT0<%sRyrI5zo)i*pWO^-`R
zK+x08YaMgDukF(2$J9KgqN7An9biqT9^f;B`oXxf?v=JNYM1dSGC+@ekROwiz@R%7
z{^H`|e2I67m^cl@?DNwrZ8g6T1cu-N5X6JkF8UMf>X<!>R;&ZIq+&9b{DZu%V+VHC
z1Y}lKVZ!8<l)4P+M-FyB6RG(7LFa7l91lgP6N!vVw`Qm8=ZLr*-#Tk$<BQ}fibWA^
zZ61>KFuO3`fJ}&P-5|p71cVA^vKh9X8*?KgKK<ia?A$sM5)!x>b40E(8@uuf3XG83
z?}!Lt2(gn`4xn!FBRN@7S(#jlg^exav&WC7CZrIF0%ckf@_B1l2cz`!aFnvVycQ&O
z{`il-nKg%uO4h&P0H=4}kG~_EnlvlbhYg(C@X1VwwW0FR;bD4)BcJB$BUCiB3#W_C
z?yhZ7YlTA`PWIJcadJ%5u2U~j$IyrN=^j&<$uW58q(&OS`C2ok=lH^I<w<$F81w-{
zrZw*zMB)~TWmWmEJ}OPaFyS`4H&OA%Q`56FKe7I*MKETl&y$_GF+C=VM@x89)57pS
z7>s2k0F6+VUujN+SthW$QD0YPi&!p1{RR;H;+qa4*4UmRo3(xr3>;7Ak8Mfev7kyx
zNQ@2-)*KkCX)6ooD(;M?fh^KhALe^^gYDiDE(J;qZO)Z<8L!3!W;wmN8dT3|Rk(;A
z_YgTaICM9({VRfgb$9>j7RIR?M9oM7A;e0%9g{&Zf{=ez6>Gd?53{s}hK8$nx|%|z
zQQvhmDZjT0TacrQifU0&(RxOqa#3ZagRU;gywZA*3o^nBkpDiiJs<W$;B*mBMp8K2
z%XEsxw+AKDlasmm7q>0#;r!GFic3m1*)2t7WJ&}2VwGNmj)2;OVxe+aVYfc>t+#K4
zCXdBAdB?BU+hcvq);&ck3^;Y+YCQ%{_D}%eP;r^J6#k+~ZXJRDJo)X`Ob*OKnhrL;
zU)rLgqN=I}+gh7He#BGsBlE&rz&<I=mrXa{FHnhBDYDEEkylk!RZ|03GMR+6{}}AR
z1<tPo;xLy=z)uPy@Ys<*TAp*lxN^#K2+MVie=!*Q%AMs3F#q~u-ujboC`j}Xjp9N>
z^Fz(*dm?bZ1!c1s_3iEM)-aOL;(OWH@HjLQ$fb7=R9V<JX05GVY>y6%4OLfGq9MQT
znG_LHa3UUj$B!LU+DXFgJQvV^CxZpTQJ=f>+=rWEg~F-W{IPz54%?Ql?1_fM$7`hT
z6WTh)z6B*E^bw)X+&ZqDk3$_`PR^guUS3lA5&N#p*40(#{NiG(4~)>q-nrsidnWAg
zxk$d4p{>Ek(l6SiMEIGjq9WjJ!&=WX>hA|L1*dG<`E0>JgzD1&6*Zp?+8*v8IG_5;
zBKW>agh}S&6Q)cltHJ#Zx4N>jkUUU8urM<S6WG^N(kU%0>@jAA6I7Ec&&tYbE33<$
zo}QVBBH}3WiY5W{&5eyM46bN)umDt;1w!EO+x>M?0COD+MDe|DW%pMHP>}B@)O}!K
zF<4#wfQ=H|rEoNn(|2)eX3Pa*gVuZ*cpyvgXKczxGhgk2jWb$WZEYR6Ojw6MKBa}V
z&ExkEUkjCKBSlj_E%y!XU!>rPkY&ZhyiJc1Ad;e1lXR@g1%W{$7vOv{VYKonu0d!i
zQhWQb!&OAY<<ctWb+W$StIs?&J(Z@T-B!=Tkr?qEk4cGvfq!WFcXnU~#4nX~V`Jkb
z8i{nSA}IX)*&5=RpNA7P6cRdF@j2z_&HNI$vBke2m@~o4*9_+u+H9t(O1Wm5NW|Wd
ze1~|4A5v=v&C+RhrB85-2x2Iw2BYk9f0+>AcHY(3H}@mevhnY$6ISn0P*FO9w!=sv
zsdmd-!NG{yWt3VXx!En*6&2&4`qc8M4OF<<%Nx>CboaqVzKvzvoSz#OR}?Dm*X;3s
z6pbR6ircstxtl$B0USgTyEwbRbxclPfMNmAQM~#NAAc=swn_E|1WIGJ%Wxfo3!2x}
zQ9;*aam7Vd;UjCSzq?z{I8_}^`--lvt|Dcm{|NjVijn#~SNwaLnu!hzSp=+45yf;;
zNz>u(1@4Dwhetl^w`X}>K_N{c>h!&KWH<XqJ(k*Q-jBBvJv}%8Jw1br%!nq7_cvi7
z4iyzN#zdj4{!z()9zCXVeul<F+F$l<S#ED_!wEutp8&wFtM8v^^eA+8g2<e~n5PCK
zslM`U2VDo3n7a>YvNxxv2!Py}Er_*@A38>8sHt~nTD2@K2Z!*OJUu)>Xf+TP3jKIZ
zTUIta{0}-VE`c^<yyVsSc_)@fd`oI7(Ic%2$FxSZ*^n(VO+i;euP&=s>D?AcTzdOY
z>KGykL0qFSF(CuR5c;fISEG)nHe0Z3&Ly|`0aW+by(SYLV9TDKg&@MAQKn7%WWH-3
zBaeNtYTMiAY=T?tmhr4iQ2V3LD=GN}fOd5yaW<Bwvt-WhsiCp=;ARJxSyDrOu@P#^
z*Gi2o`PIkA@KFiX>TC_xbl-&5UYc^uHaQE`>&Hvw@j`;WCg62`6*|K8`0$o@5ge$A
zPz<bu@2RQAhL%mXT}UX1C@3grZm^nfllgLYcZk-|N+sHO-9|^QBfh=Y{?Im`#bUP;
z9k1)~9yAIH3NN>q8H#wxo8x6He0+AJypvihh^m^(Mjzf-8V}Fptldu8vf}qV@_b`s
z@9v9yataIr-p#pzSTg_9o84Vd+4!5$)ohllJivv+9jKtIS(i1yRIk_O%)(5~O#Qhb
z^F3rD0MXc`htJm<+T#oL52T>zkux;n7=Q#X4i(?TSt!8?YKVAj<T2pugV{{QSY<qJ
z+wrYwabekfrH__-H$7!6&FX94XnykiVI?ZL`}>EJ=y1z|V#w_&oUW<&qZ1a}By`3u
zQ=Yn9uf56hDm)d_oi{x=ck4C`D;K}cYDX3TT@1ZuTVkL>WzIG2Fi$pv#0mDh(n##)
zSfE4Ru;M2@O8lQvvFO9$l3FWOy;g6o&+JnZ19uyt7-n)ANz4X@rluf#Bu#!TmDg3O
zRfm9p5QWcD1(w*0-be;MyLmDL9JAq{m2RNu=_ISpf8}!ZK7O4k$M=k^7G#}w9(UL%
zCg*W9BRqCOLM>S_6EjDP6!-}V*Cfy(3Tg7oyXzw!dfhr~;Ip-C=r!?UkAJ=G5-;TV
zuw54~9@M3!Pp_`nSXgLiY4{y4-FQX8gb1s@B6N>$nJw1^&6Aj`^7E?mZI?72uWeA;
zsNNZk43F}?xOY)BB#RfKsK(3E_VyhCi~j86y_JQ9g`^{9ysVWqLv2ls@nk}&H9FP(
z_y=FKjA7mIzLkc)N<PnP&3a!S4~@IMSgMcP!z7>0$4gT<wXiX8{%I{Mv&F3Ze06ei
z_;^XfL-WC*D{my8cLn5s+>0*h99E-sHd@c0-CU0!?|9r!JV4DrQj*noC{v_MOvIxn
z@Bzd1Xk}6!db&H&wc8cS(V)##zEa)Ul@d*}R!uf2Ib>)^!Vh`2o>-@-HWT3ui*Rvp
zov(G%`=YxNOXc9{d0&d(p4{irGcB?kicMQvP%+;5$FZ|JOtH{;C+f)2a%_x=nU!^O
z_EfJ$91Rt{SMT%-?%%h~Amrc@kBz6-=<U%J61j!5b#t;@U3nCg4@!(2@2SnZ`(u70
zH2;krw`HvTirmV?VcmU8OPQM$<YtDf)=j~jarP+=eCLqI%BtKR&^sX5Cpfi(6_J6(
z@DUPXXfTOm0q1YdAp?1h1|AVGKKqjk-U4!cInX}s_x7V0SnY+0c~4=IM=;gH+KL)f
zWHb=)NAdob0SJp2k3j)Rj6h&YbMxU2v$3+8S|NwmMy4<se+5L~5B|D9L{9AFE{jnj
zlTLPCk*SFZj?3w)z!JdWQj3=IiAk%aAv?PhOskOK2hQ`0-HBVSfIMEvH9Z$s;k1cj
z;gp|XfS!J<{aW-o<;iDjtTrp1;@J|Ab(?BymxMNlrn((_V-S=5wJIk<GM^v_>nVZ{
zuvgu%ujcHU_T~jPR6Jqok2}+JoOVmMJB!TLx2IWMAx4-3#k1w;hl@AUob=E7k6o4Z
zQag-nPmiF>Az`ssM!a}N>MZ=%&|HV0|EIMX8m{d3m%077BifCDp$Yr=@23ba&Nc#P
zVE=mc?@5ohnO6dTvB$qo65#(o+RMLA>W=>ZzC_K_eY?-gp>$oJlst8}8e(Si-Hs1k
zY1+%Xs7UDrg}KZ5Cw{-XL>s(Z&8TVC!tO&tS|8AeSgnIn#5j?@7LF`#vl)rX+kXp4
zntiUa?dbSoqjI(S;VWpMs`3hG1^YaAiZC!RXh^m{KjmcS$Yt>D?C*=ZOKf(E@OfQX
ziY(69&0S(Nr!{&sgE`o{{Tx&@Fo^!SkG|m(Iovwp<G8=c$sbK!u6Bc}y<TcDv6_4#
zBQ2esofDnGyTxV{vR+ESZT)lI2jSnt@QCrnuB!?kHKejIDc3~|l(wbVnb|+FvqpK?
zgJ|n_&-C)2!TaTuS!eF$`DqRcmdcWnnuyk%B#q{1(m(2hgHch84$($MRz<`_sWwO4
z?cx%aS{ASt&v?lknx7uf6+VvXX;^}RaBX(aYdlpIl>x=$l}4wV(X_2%)mj@7P)bf-
zNG_myd^xe|7>Y`W`<6siQM*L^nRu~Y<&@S~&&kQj$H!YVnz%TxC_FCg-_y6?_;hkS
zpA(|@v~Lm~99&CZ+17Tw6|o7@Du9}XVP@tJ62RvZIEbQaVSx<v<K8W3a<_zs7ZphY
z;&NWIlf`leGcd6zVgZl!hS=(ew)B-Wp89g##uorOdenzC18Iuw*oFpB58vPlMJHSt
zT}l<rx3aLJVTqe5hefI>{8lzRUTMbwl;c!<qodk<4k*(bVSfG~Q2M1%$i>CASZDJV
zS;U=NEzs-qbWLxj1muwkP-dh+7Rc`xR&)m~EiGVDR8vtyBVp|x9Q1ssD+QSKSKkv7
z5<Xs>s}zHt&-C;Z{L|Cm(9l*~VPP(p*C?+HBPSc3QsJ33;yaKro$t<|0VwAeZnjXy
zHz>}|E}YhL-P;vl;J~e34#a{c4jk~&Xv5D>5RI`utL?e_Y-3<R0(jGCZ=h<L5L;cH
zP^YRWwC?}(z>G@BZSDSmeiR=c&%i+c8U>}!dO=x52N4j6B${uB0o-ZnKT@5kX(h>G
zfR1qk(Dq~(>L~ZPzSNiup(*RyCi}i8kHQQ(E9Lw?g!}uh$8%?EU%3H%eEi$Hm-vK)
zp#71-<wScuVFa493fET`^))p=4ArHFyIw;c;Q+6jBgpw>SaZrlu2dq3+Zh>Y7b<-S
z4ak!0-rQ%cU!$O*IXSv!O_;YoUlRhxrpC`ru4o!M|7j1NyUZ=I|KFX?*TEwHogom`
zLw_-h!33fjGm_)f)wE{WD_crRT1rp>Wo2b)>5<nc=Wv2DI;~}GE6deh?fzeh(sWG3
zo^s>H)Hv9uCf}h_m|Bs*bude-iQUYCg}z*y%OjjqS6ouEsyupZPl*bauC#u*?RhvC
zvn7@2dw*#$-th&HSJT@$*w{GOi2eSCX{a!(jKyesIDRN07~!QtL7YGQ%b{oT7)jc}
zp##GG_66pbVxiS$qqn1}3IoND9>eBbTx#X&YX=)2n|t1eI&4rGo0x*ydK#b4LYb2p
z7uU$}u=et@S#W4@Xh@6qS?5CDM<6aCj$Ws!p(&35@15SG5_R0X(>8B-7%~Rhe67<?
z$-EP&i~t?vKST3(lH=*V!UtMgTWf1;+Y>i=xu<gZER4CCg2{Kc(g^5S?lS7|^N+w4
z4M9M3<lqQJB}M@jG#9{O*Q_%C{n+RPN>3?uGGB->!v#Y?=gq{#<a;}Gx>)T+z>BkP
zu*V|3yXWrV?z!%GwxdH|hmQ~V$!tJ|Nd49zOA41^f3aRKu}lZfPckRB9Rj&ICN5uT
zO=QqG+&%^sMP#f2@4#^koch@^9iEhAB)!MjX_xyc8a>jCxVX4-ZqzIb-ewmz4i>c6
z$j@7e(`j3EDF2hyPR=n}VPn-^<-YKt%IAZx&o2PMu&iJC<u}{YLl17Y3WdsAJUjq3
z9n+SjrG<r+Qi1YoI;%ou$=pAhYC8X0hb~J^J%P3rQ<npNhllhG#5Psd{~@g2-``gW
zsHmwtJlwT;Jlz+U4&MGeqv0V0{w3Om*Vb10+;7qVo7>x_#>V{v8ft1FIcaEYu2wHO
zm@TKEib*dW|1<Sxma_o@BLR#}Otv>S=jP_vX+NpH#|EVv3QF>RVR0#`t}RajyxECf
zBp_8so1}*q1S%}3%{~_i#VI_1b4}&l?b%lmk6@6khI!1@h$6ZaG`s=?v*b?~-$fge
zB&F0FRjPw;mz$frLHo|b1Fo&jBsUjWJv?-Ly5}fV9>hB}GBN^nH{d_@@^7V`l+tk<
zgd<P#4qmlW7&t-W>~(c@pwVYosZNf~2YDv?-wQ41c})3&{8Pw)vCICSS|_!K#6p@Y
zG_;;z5e|-N;QQHGU{3XGU@Ba=Py$)!kk~?bp$}sZ7zaO^_Jf4rp1%Z06kvy^rjkwU
z@VSN|<itR2GZhOzibXj(y2cp(<W78G<*3CeEh;JL7MPj>b-lw+3iP2lv~Nz&#;P!a
zuyJsXmMV?rPS>&w3<_xqA8t-c%gB72{khfBm{yJOLm;5_sHmm}du%+zZ1V%H|EZt1
zcNB=;{25)_0PvqWEl|N_4W~Cgr~HL3NJSf=h2{3^OR)FDu<~cuw!N}S@C$!w#xFUs
z=Jp(xmX`cOihs3Re+ce>i===)zG$u@GS=3usFQ&5k}h-r16c<20>LTnm;T7c`EN*a
z8*_Vn0QvgSC`P`2Lj{y+L8rss;R+Cci-KrPtiubBC|E~MM@vsl-wzr=GBW;tOm8D#
zA1%*MPcKgo>)a<^#K^Oo9)1SrPZc{mJ39cfTEWHP%@4i-%4VLdl9HuxRzx9xln19u
z0s?|_HYy5AN(#!N3Ki4a;ndF#=g&S~$ki%mMW3zFLH+Fr#EH5JXLrO1MUi$tHY&KP
zII5G27IuV3%^Lr$@0~^DRS}+ede;_)M+oM91=l}HFj>50{7*?|?p9F9rta9TFmc>F
zI8akj2@}qi)Xk+04Qw}%laT={=-HEZj>z>riLbadfxqB(YUwAm{~3$ui{&v`aRryF
z&fz;rNoCrexw&6?O(=iMU5$$i2Rj(8l02`6{`@cm^jVYOkua1L3?}!S2~%XbGWeg|
zrhfgWtU&P)o7@C8&(x&7kdx7JJ1od4ehs9jr`xTxAOX6FV;vo@0CV=_>If-1g6scL
zF&5_Lgk@#L#KcZ4CILa}H`|-r{l|^}OLJIlz5(fgh)A$*s~5arC^|7JD%wh`SE!k*
zi<6QeElXS+QCb?b)hB{VQ$uS5<4?1f`)U~S88UeKMlR^Iwd=v2K@%mx*1)2$v{WWn
zv9-mM(<UW=n;hF;8kFCbySlIdQ28<O9xf;@2aPJwwmexZCk5~c2+E)h$nB&tKYudY
zpT0o}?p!Ol-ksR+c&x5FkF=|tt)CTnm1VJ#-|FVzV#JKUZ@I5`n-x)P1$Gc^Sd}OI
z7qb4Z5r_FKIKtYs<d{@cRA_@Gb0?9|h#4`%4M~<pZhw{y8(tkxW=Q6!p1U(CDFK;T
zZTvLPjnxr^k)lj=bUv_S1^^7!+*Ys2Ry8}?$mEQal(P?|Xp40o2cKNfT=FnuMRU0t
zKjYzIGip_-pABu0^j+`HTP@#TZJBK4%ccMGlg(>2^Gm($3Iae7oR?MhI}>8!FP!@a
zbdS7!o}{Iv!zII`iJpjoAF;9bS0}K=d$bnGl>;><w8(#ej{g@GfPslluf>`Mu%Ams
zKjOKKfCt+?JdO{Os`KSW(s?*(x$j2~W+lSHlu`E3((3J?*Za#}v+1pd4qqiC23B-3
zAP8C6E$hFZUN*a)SQOZ~U+x-Qu62PN@gRW#4i3b4|CE##FSa=39MXhcUe?4HtAf0k
z3S3VV6zgH-dacP^le1G(HNSotPv)UDXjA6p9elFRl+Vn}d;|L({9}DxU^GLo-XKu?
zeT(mt=EL=oiH(wcXb{<({qI7#xH!0)>YB1YKP%A2A1_rtRTEk-U=!f-`#yrGy~1l}
z6x7Fo&p!8Y8G0Sp#5}o*d+~IMlPmSZXPaP#!LCnScWD)u(fR)^SYRbnf|lO{<MjhN
z5j_o!FD7AeVJ<~+Okwww{4CIjG4XH0^bLkgjr&O_2KQB%b_4bEe03-W>3fyc;S|!O
z<R6ZpL!1{T1M$W9cNGS`28{x7M7<7$Ex->{ZX+W{`l3l=g#JaqK;m@<FDOQP8<4sB
zy-epwq&C~hY;1dLi!Aux_XN4$OCSyQA8oDwj|2bN=lbOU@-Nu;H^P0r+q_{J3F_f-
z0skftAYaBnTKLZ^|9@U+!S_jEqIdu1H^=~o4-GGNb`3cDQ1uqO(sH^1zc(+NwTwbH
zV8@Z9jgU;d^vPv;Sa$^+XWw&8sT_uli7K6C{lq4J3Txh8d!g$L3{fOHX|4KXJy>UF
z%$DC_3i9k-S5)-)NNaulOqbZOpjj1n43A{Zd;f6!dgo|)37H95v!%b(jp?rEnb&52
zd!x@4=5U-C_|U$vZL(PZiD7(v%6MY}>XypKH|mzR9&qzy(corYa9G?Jztx-5KGgmz
zBH{wPbldGLrRtpbdO?)`mtv+JODh{5mvh3!@m5H?Zs1FrA4+|KP$oy+N$iholUEa=
z@^fSt9$uU`wGTJX2c`F&JCL7En%#9uR4F0lC%^xhufB?mOvyu_*9I>D=^Tcx6M~}W
z$bn|7-B?0wUxE!(@+b6{4$qy9mCoT4ZIIERThYNorTb8xT@Bs`6wnrs3mw@31Tgik
zJpRs`x4^EAdO{JVk-b;7MOsM&f{=!xol;I)9;H$_bZeILabeP@$fl34u-rAjWhL#L
zVC>D_{!dT^2CR{MOgLvLqBHWa6A|YZN1PngZ+9&YG=C^7-~R%aU#g<^%->2{q;5#)
z2wSMFRtSxor{%9Ujr_@bx7p_7$QmJ>eRY91rK~!jXlMtGYW4A&S(q-h_2W)lyvbjq
z3-~$|9)U9eaUZIq)eSzh`XYP$SU4H_4{6`s!pLhbae{8lN@3shMC33yfk*_u@vR|U
z3i9ryhP#h0T<}VhKLTEeI1(f$Kf5K%WXb>f1G^vkG6`*2fh~rR4ro1FJ?1ZUj1O<N
z@BJM;5`1d|D>jynb{E7ti9;LowV&r{`}eXH%5fYu3%<c6x9WFK&U4=A|0RS7r;H~Y
zz)F3ZAVo#<KfkZrJj^EWl-JoB2}uSQ-~#D**gX4t?5*8GUza8H%%p@+fQH?JT(#35
z;gWEx{V6w`q8Liq6Z;fDRBJYEJnq8k_hghzT`SKH{#eXB^%+FzEEE>&+<vd5;h+AZ
zW_IGIdyqkO#m?e5GdohcEq8zMDNqv4b(goKH3>5^L0-G+SVOU~AVD^Z_vhz*&+Yu=
z`}+)r#bRdsz)_vTlH!+u7+)ISlh={7@101b@i&A{wJ?b&u*YNEYj^y5Mm4m~8qp7o
zM2)$*)eN&z9UEh?$+fJ_JddfoL|fK^Kr`9*VyD>3(TSB6#0QR!veB;&4%a1fCvdH`
zF#xW2?|9To;r!AAk<jruen{qQK?b>MJe*495dZ<1^eLlN0O!-gZkf&)X?K63dB<3e
zCn~;R{-hoSMH-jAJXs9bI80;kf%u5~VsOZ*nl;4~H=+kD4+lj52xXy?aL$yf$z{q{
z{h5?ITGmt3l1gZdjolcV%yU#$p&O!;JzP$Vjq%T4<l?#};Qdxw<oR^i_!6QE4^T#M
z{DzKI#WDm-wqErGi*eemaXmy6O?b`86GSFFopntdH6uPI=P@8_w`4MN!v27KdAw)Z
zq&sfF87ZeJz|H#R_(y;c0505Jy9?YnT-+BO;HtJ$uPCUgus6(v)cZb>8xB;2VF?SZ
zop<P0T{mCvb8%6B6U|@iqo`xlQT-nK8o&hEzOQ!u^Aj^FPnAty=->9@*>QbG`|o*m
zQJ`el<m9&faN{^Nr_IRPGpgBq{~TXT;&bnO_$NtEr<$Jm6%g_dX@0ly%=FCl!yXl7
zY$F+YP0Oz!>Clgqa*q2G#6bAPc}~4877Bt7gur$o3tlRCdh}1x=l0c(kVw$1G3H+K
zJnzok=xb_dFh4jDX!AHY-l8++A5&D&t+r!|7sWY4l!6a9?QRo5<Fqi7&6(hEu8GH&
z^zzkFQ&I5Gzac)0t;pM6t0e>{6Q9cmDX9!pqKtgZoE`1s0@y!nPxR#Ufh|l~P5s_&
ze`iLiHHltT#9}^cM61Z1x2kEXap|3tsm0#@i;i3)Doi7z0pahH?a4U}t*>Oh<|@{w
zaosI0-SJnOSsd);^ZK|t@5r)xX)Mfpgd$0?(63DxX~4h0Wps~?Bl6lo2W~}1y%Nt~
zz@+EelV4)7HYV6E8XDX3!|Gk|DLrE2E+4Mb+?(psj{_nPYMou6`ECEK)XC7sW8NcW
zISncpsi`HBz`vv}E_n2~M=fI%YV6-9+pH!%zoU`UbjnXDS;VAGU>>D=6)*Y3a6f*`
z(e8bJ`xc1%IUFhGtA~TDtDsrr{vpU$yx?n<?IIC_&T@~|r(*j#_wOy<kH&{biwuz$
z;nDFy{4j>=s^dS6Cvsi&j5>@%7Z>a&A@?E0`LbA}aIZG9n)34WHhIeuRJWUNfbZA$
zk<!TF2oRx0PjrI0N-MQ8%KG=5dStd0F3z7Te2si;j%rTF_xEXS_@1cTPFzTtSJ!H-
zSU7D#u~^~@9C5R2?jLiBXd~F|Q~#{d3Djq0e5#v=URgv05hzRJ!y~;Azs^_ja%*?L
zKJ1J!`kTSN)#ZTt%)^Rz#kK`md0m->p{0a|qyr6r?>N`R{d4&tZt=35-}Ms}G%q4o
z5nxtO8Yy1e*n7MWNxnz=ir(zPV89lKI!AjeKoI-_@Zfj>s85bCG*@lc?+jo#p2NM5
z9((Dh|1i2Qy*<-C_0etx-p`HdAna3u*vhX`-{jST;@06VM4(&IqEc|KuiM<7r}}I8
zijB1RTMk)iil1?MP<f``*fuFeMNZMSOjwwkoAY(fcVP0zNy<jcU}vHFXjBMU&B&M4
zaNDD&+BsQtv(;|+rl-RP*}uF~%HO~u;6RH!^KH{gpx4?b70*@lL?a0>Yja%R1h1h-
zMFFXW0}~+MOY~bC-%U&yNbyEix<X#~DMcTP=H(5&2y`vcoUXipYEs}vP#8KK3@=Pi
zkJOq)L$}4sP>_vSH3EC1J`4>^&zz_6?3^apy!JNbZ}IJd{Pso(4uB2JwJaA^e6tx+
zi#kh^HDd<!r`YW%V{3V-jhB}^0{i}9IU0uph?+e-FOM#UN*W_f@6U>FjZCHOG@s>S
zDGf3Awv77V^y<LI0rSaioG%|}Kx;}kXz1<QR2w|g`UVpb7b|4nnJWSUe2eyQUh9v+
zJ3DJ=J?#+y3arehlvNn6ui7D3V{erx*TO(esm-0Fot}AjH%bS5GQRig=IYfJnUb}H
z@9%~rww1hJqe-kT9`Y6R6OCS+IzCT5Y)?XcQC+jZ-(G*V>YG#bcSKFR%(pg{mM>GH
z06z(4F}Y|uDp@hv3<Xt9R^uggF7EDMm<+_rX~i?vdUb|VHHD~om`P)^C?Xg6vd}tb
z5hVi_<eBM_qv(%fpO`d%Bt8vToxTU;XQzm^j#^y?s`R#ptDRhbw5RY{CUeG%YHc3W
z(~=L5&JQdPMM&!e@t2ns?vZlAc1M@NT%}7zC5IeicJ^VTF98>a0qqrci^<~DsXR)&
zBxOMA^E6S<k%&0~YL|Pi$)uS`Di=P7*Pl(JX4iqr*{SjG5$&WJZ;3Q}ij51Yhaw1t
zeZ}#m`_*}@&_PFXqx0_@PGkUEV!>2arec+dh@jzmY-}+bHq7}dO$ojCXt5Lp4O(U0
z1CE@^^z=a0+ZmB>d3D!vu~atSv`>dhxI9BdZE!G<ainl}a1gpG%x*}X4{q)%`Mof1
zN}@?MYi_A7zOL_o^G^SM{^Lhppys4Yd4PlNS!I=ogYFkEoUN_3F&gG1dYn&$fOF(O
zD2`48+d%i3{rc_t<0FMH9ER<Z{plre?kBoS802VxDLvM2-<3_l$k*I1F7EDl(iYg9
zg3z^~p+UOX*jyBlUzFuwI7Tw7+iDLnhfbJ^$oD6ozPjOWUDVKsc^5Oh14-t+`#mk7
z78GQoT{Aa2A@NBaUJxys%F;0%`!Bn&=@htpZeM=#)WPSvmC@*VO7+&r<`?VHm3t6X
zMSqSL$(fd@c;2QQ%lk(md%}_e(}bYZW%|i_`gy(J!>xc0jtNjuR{YYhW^j<Aa;(gI
zxKr+(^6V@;AWv=JX2kf*Y-Ml|&d=AB64CqjS-r&#{}vwuwJYb8Qr_g*aFKLwZgfV5
zd&;wzrl`)whiklClc!N#{>1CE;?jK%<5Lh7bCL=S>QhtjN7-QYXwEHA_VK~_1*B?w
zJLQQetErAAvo~g!pmtS6+pX*q?AF!<G1``mk@UW@v}8t&X!Mx;-31TO+t}#wMuXk4
zwL%Jm7q@wL6o!=zwFfZ9GDf)*zBkB0_#r(@4fFE@G4y6sFGGiGY90@*0r%(Awesqn
z!32+JJ18iy`+}W1I;CJc4FI-Q`?&w@l6~Ay4(f|0T3QA=R;G{c0`cM6k-93BUZSTi
zKm=7Alx}S6(9uKdZF`1lDo>Kno{8J|A<hS`JakrfbQyMF^z_x+!V4}8sU>@Q0-2@x
z`F7SsPESaHabB)UT!QAp!Veu;YP=*vt6(8rk)92$`d|2zO8TYZzj~)X`>g8g=r91Q
zdqLbK<>g9XN8ahBew8}LbIUa?3o_G(oC$V`=nZ;no6Q4emQUya?ckuLh>#1|4D9o~
zvFH}Rg#&y~jPIGAhVuh4;`}>OsfciJx0TmJ)Z<WdQ&W$2`-h$i%^35qo1M2}(fv~*
zyBBFh#ExvNr=t8VuFEqMaC+fy9Pb6E_MTokeEhTP%z=i+`gZfOe4E;qNeC2pO64zU
zS@ytNRSh8@<&JJy?(MaGlLP$nZ%#PVy6N^qg@6CbCFhUAI<?xjv=Z>SEhrTu81GMw
z{Rb?92KcIRDO%rC7Dorq7kM0C5~dttkkSs({ZiZmsqz9X2`26J;>Kfa^NqIWbw}%L
zt6I6)&2fDVf{=i6%VHfi;zrB5w^yp2ZTX*ULk<@g?m!j_a`WqjX&}?iSUE0{X{<G+
zrwio#l-;~9{D$35E(Eq}Y9?a_!0jds;NP89ajGOMV}hdv@*wvX6knIf%)C5m&icHn
z@EkCnHyfzhN7JZc#D!lN=&II5`ST~e1-A!~W4byxaWJ#2sm=Bysiw+AB(8jhz5GkS
zvt_biY|}wYkrRrhBA;E-b`0A;@FU6{npvKcw~M=WwyTnDSH(FNS&imH;`H$&M6v2z
zvu>HGV4~YQ{@>NqUPfYE3yeN3`ty0mNifp9Wq(wONnphipa=KG)w%$gzm+k=dK77<
zrZ|7a#novdv&`D1Msf@1PwKRKGAy(>h=>!m)JAkE;Gpo6CchcrYnKEu5b^EB7%0&H
z!p|F=c27Ld&ZVfBPaW+_ZMr&o4G8AGy7LC~ojxN*V!eH<msxZ@>zcGlcnzCwAb58I
zUC+3io_2Iyn=7*H?5!rp58>dOr!$vM{6=w$OPA7>#AWu0@;Bhuo0-q=g9KP}L}hMY
z_(NEHGhP8eocb{Gvv1?OBs7-{tZ8=k+iA!ac9?L{J^=%_?jl)`H$Ok$4{>sKS{y+6
zGZC%)%rWed8<~PqNhMI5P1rGpPBl5@Z0pRCMqKJOO1t}cfP83fut>Yz@`_ux+$34&
z3S~YCv^rD|oV&*&+9Y~fq9||T@SqG%2s3gBxQT#YkFJ?0cAUDQX)d6*iB@712R7ZV
ztYU($M29(`fR{B^Trr=QfP0XQzP@)=TWh($k7HsI<nk^hCHgYGq^Qi_0FyReGFP$x
ze~@lq;`fx8tNL_tV<zr2MobiDRCvJQYhXzr^>Jtr80Y=r@#5WoH8NY)9rB!}GZyPF
zGLh13(137Sv3}TLYiB~Jt)c+A;$eN?UH0Q+=Mt|MXMK%Y`GSVl$%BoLl+^pnuRYB+
zL(76$pP?^oAWmRtGtq>^<KZ2E&G7FC;}a7s_jk?r_Hl=Nc%(1f3x-XHf=WxMtt|3C
z#=Qo*(F}p_lDU{+!r=zkw1@yXxK|2LEEJ~y{Y&w-M18u_m+haC;cmmq^sE?b1GzU|
z65h|+Q)>oTy&@%=G%?@G(|NBBe}DZe_;lZFHkZR5lHcLI7A_we91^L`hx*)>CWcx+
zK}XbKhY#&_CsoCkO@xHMRFw02@-1nq=TJpK&A&;~$;#63n{Bq*^z2*mndc*i%kTxy
zk=_iF)vBM0g*DQbcSZY6O?GPE)aiX4Z&v2s@gwz8{#1SuS?!%rB}pd<av=KnF}%S~
zs5pHSlaqy7^;a=5wL%l)XB+G;&RUv6CL%pY0*uQ|_E{F$;?(+;gjzjMTE#`~Es+tp
z^|CJ0Et1}NFM!XMmhm{e=c&A|8w2*>UJ!jl?Vl;3<<hT2zO6Jd@H}<lxj0Oz5X$6~
z6lLjZsv`hD)vLJa$&BZJfb!95pG+|wUFkIc&M!KeN@exiz$nGAILVlIH$3UD3W^E}
z@Mf8=FVr@BtijC$3MCEx`Z%-W<C)*k;Z{wA)`x<GP3i{Sp-CU}rTN>un1KG3u-wkI
zRW>zNVqc65JY<nu1uZ~+$9L<zNHV=Bncx}or3)jTo4%0C+k0;n#FMTA&fB}S@^v1&
z(W3~wjpL;p`NHPWjCi^i7BDhl2QLoJksZu#h!eb!7ftm%y$KF&e7FI=l*T)OR8-(k
zZ?NQKl;AEHked%*c2kWD*qc&OQX<O}Uu{|XlcFHd9?mqgXz-FNgHh-Kd~5dVtF|LW
zV;Gq-8{607!2Wd8WjAFI@&3X?q;~lSer`i+ILbLOAD<&Dt7CN)@BL-j`dRXdt!j{F
z*0*W=@hSEO2J~kr{p_og0Y8KG;Uub_ig`z<ob>jE5I@X%Y7Nwj7_4nN6PO^da|&O3
z;4d73hMo96L%(+=Z7qq&NQ8-Ip)zw0n4hz1S(S;JZs}<;hKW#^g6aj5Cp(HslTKe~
z;~g6QOp~NdZ4Kq3<{Hd1rS5QI&@00cZH`vSy7VC>#hu_?_a@FK{qvPCb1GsF#|>|d
zN*S=HKgaJc6jKFeDJ#m$tLPS*nZE+q2}&c1rRd0?ZlBYnz;4*_HX|{3tk?wIva*4#
z`1olQ5$h;RHBQb1$q0zkS<i(#2(!-6DvzHY+(^NHf56`+Rqf`K!-ik9zptbHOqqz1
zvb@q1-`;=A%Ktk@jx?c)HCXbBfc5j$yFH}QgUwA>RaMdLZD93JaVV@45x<eC)&06<
zY}`PZHeNI*GK~w~PIv!e=eX4iOh?MJE{D<?a5<uDNuYQlqcROrqNwEk$o8Tmo&AXG
z8R9D$X`sXEG_xkL{B*zpr(9^P{|&&+!^6RMr9+4LE;;E1xG(;P69EwbZq`YUdAq{Q
zWhPVX-Vz|k)v(QFS;V$8qF@@kB1IM_<<L_n9_lJ9o8yxQ?<f9#7#0^O!4FM0hgN>{
zWnP?^yy!~dbacMu^XA_%3Q5Wh=1!E#Qe1wxOvvE6ns+My@D=RZj|F!dHZBd0M*T(i
zuw#*;#w{}e_UekC*!a}tY>phSoz72v5ddl2!k(?ma=S{rEQ?nkYScKQK~Z92dISgX
zL?;R22pJjdjmjL|s;fVy?!!P13?tenXvS>bveTe)l4aFy^?czM<?rubD52`founNe
zd&fVrG$^Sg7Al$RdAT>KdiGtu5=ycI_x{X5o5!jc)W8H9VY34^Wpy*4ZVM8$I>uj+
z9CWx$Rr1Y(!Xl?eY!s2_MpV`ewePq+)ICQkaVaMY#XASxarHDzOI|k#klT#!htZ^m
z%i3@qOM`#<hoyDdKY|8PqEy-t!{8XW!O<cVo$_e0vL7BVX)KgpD_npBL|F`&D7kZb
zX%5eJ2T0L`um5&QnJa-l5d<d4<iw%E9p9ThX&XtbOaDfu)%WJI*;xl?W?&5-UwLrR
zaV2CYLysoEQ@?;2KR3Hw7CyXt9YoruJc|cK)a~>%dJpVb(+m!OW*z>(9>-U!1J+-1
zG2kmjx5#cV<6~LVzf+d9_Fc=nEpLI37R|Mf!<Vcnhf3_RIh&XN4g|NX4v#E84VBlD
zO&t8{K8Al_TBxZE@AHD4=eAY81>7%>pQ^{$0sTU1rsXD_Si`=X!>#y);8JyPiziW(
z#shk6k3WbtK)GdjIyV1tdu-AD^5$~3T(6f(Ud#O}6{zok+yvKlp@WXD-FrRgWQnUa
z1l2R;oXqiw8<01sF6D6}+1R+aTzPse7F8C(Ev{zBtU-?wy#A+J%+_=!D!y2kisD^T
z62n4GjFA6&KtK+U8@ym=j#%Bb4ul6=MO88N_SRt%YR1aWc9q8I_B^#$%51fNVfKHK
z_SR8RwQbn<AS!|iiU^1Tf|PVgNGS+Nmy~pOx1@p~ARr(hB_h%t(j|=u2uOEJcMmh~
z$^FFp+_Ap*k8hT)<#H`HGkf;F_IaJ>ar_Sdg|uKUoAWr-YxCKBG5PQ2b277nMoRA6
zPM;(3)mB%j-58vlBXD2)F$}dx_Q4uP=B|@HJE~+qcya4ktcAgOt)5CvY~*CN>VUK6
z3mx~tda6|4y@oPIEtTe3QJc{PZg1wqZ*(loyp;w-9~Zy3E5ZG8a7%C%et`PIg_4#S
z^`?o+S#U*#7N^5A!zE9XX|V*ZC8~T1-?AiLYJ8k-QBqk~<Ojw@=%uCmoxh57`*OZ<
zUAvNU@wJ4X{r$Wl!lg~P*T_8G^T+E{83|)zkg2h7gSVWVz!`}qidogEBSW+~KE0IO
zEr*L+<l>Ujn_+nq7sbA*tk=Zyo1THW)?ur6hAG~4e%{VyU46tkLTNB+mX`MX5t`65
z19VJKyq9BN0tx5N;cO1aVwZ}dB0kP}lN8+c?3KB8Xzam0M8N%!b#J)%r;(J3zvw$h
zjb79q(>^8VRSLiT2+7n|=e1aTe9npP4OEi%z%ykfx@QPi+qD<<4Q;#@gHUSM$Vz|d
zF*7{dhtJurG^F0AcsxNt(R|^kt}~db1Gc$_T@sg~gc(?v)k_WH+2cL;bqT!Q#K|)^
zRJNQPIj{D<7E8aMoiP<Cg*^8?C^$b!pw71MVz~U_)bfKq+_ia5AJ%>RV~S*b>{$<$
z5v&Hb*9FXojO<()x8t$O;PF-(ZvPxU7iK1UNw<+1K7Ss+hnCl2_anD1s8;N4BjVY(
z6M)wxMrxhe%>d^y-ud^Za$Ib<^>scO=x(CsgP(`LkxU_+pAYZE(B^lS9V%Ec>AJ}H
zMzh$)<~V*ZKQdw-8N`-sQ$n|oZ-3A`pE<3Un5b1G<Wc@&dl$Mrp61A}i3^1>KNk%)
z#;~=EK$CGL66Ln+*helgosoT?p=`nV=GkCv?onaE(#URA^kE(9#^jk$GbXLh?I_>C
z+K88l-$U7yxTwRslk*MwZX>PiyK_jl<q}3CbG4!Lw_%4J;@8m+^%$a?$7=%W$ZnoJ
zNiVU(ybz9pzPaP2wK7)2hv0ePYM7pW4+erdq9Rg##Ws_$(Rv`m$;sh1vORh3Q3*w2
zVj?9sQY@thn!3&dD!k+z?_|#=j1=S#6)^xM;@$kU&xH2;VaV=BO_k@z04jJ&amml)
zIONc*J|mncj4Yq-bd+VP&29DER`(_sb$^9;*aNJ6J^ihy*>w2s!vB3VN7P?nJAeC+
zze-g9M&CKwKJ}h=#8JdKJ9Yn=f(j9vqW^N_|L1{{%iIFj_|!<^KM1~Xm5bIN?-}?}
zP5N4riGjf_Jc7~MwBYNCrMYn^I!ZEGMOn33;Kdv7)Sji&V?1dY33_IF^PSS8A0NKy
z-v<@5M)c?Wd?CB-Iglfk<<$c*NA3e>VB>=2aLIi^)RA{nhHt=?!lI&yTB{YbwH6Hx
z4TDO)F`N3q`b40hhgx*4!Jsv$dO$*6;b4qDa94G0$`L~GBqha&AFA4HWIFBbnRF*y
z^a1tS?)s=|Y;(2u2C6Vc9QyO>GY5_R{Z`AnX35{aS(&n2V0-XDtJ1QfE@?wZ_q}&f
z9=T8H+7xg+1l*736r``H`)~AoYkzY0GHQj}^7%F14Zfwmo*pw0Uo%9rSj1(d<&}EP
zF1QsNWBZ-s9i(UX`r4-5BOc29{F?eMssHn%9F02EP)~}@WVL#oof36e+4`UkLU-m%
zJANRKfPeri56{nTxoEeoh*ZGCbaZqEYnIDLyYBy30!<zdFCWYE>hq1PWMKzX%<FUQ
zH^7O3C~)gg;c#$JUtkjoYY>vp{3R0L?R~P}*>ZK|Ca28GVT<dVMio7BUN8Ro?y~A?
z&1gDrX|tgCs*`S^u;f0o!F;VMmyNuHrj1`QXbDpM??#4}cL)fCO@0=$a~y1pj#e6T
zk`vb1_43$w8JbIeAFpzGo*N6&U(1mKHa@=Werh`ASJ~O4bN7{gRgk#xx&Z^Gv%MVy
zVPJ3rU5`-WC|>@ab%Qok$a`P<dWApkZDVuu0ZtFkm%hF!2PI~jSFOcbTERO6(vI@m
zpwu)4CDcIwM}9lX2h^gnS|Da6;dEv7l?@B?CUe@n>#t|%#LLSk7tf`oC326O)&=Cf
zfDF15y4*GH{WYXhMZL;=uS1okeVK&M8tH?9Y07z=J;(wdgn{5vx+|zW2g%V4iDKQT
z_cfO7`s}2omzw7z^sFbWwdas&URAn1pBrAmW74a&j-G)5>-<IMwPpE9dGr*jrY~J~
zdZ{x#5r9YPaD~XzHE|>eI&BYh#v3}dd!*!)SPq7g=58t$x!D}73(&~Le$Gp#xDF~q
z&37>|Qm(ur68G=27G&3WifEmJ#?*T9c<dLrbFSRsdGk&F#nEzet$_Oz%InG3?c76>
zSYOJft3G?SI$2h>bl9fnP`WYVbb9<%s4LMT{E|-pbe*uqI+}y{turYjV=f<;%zqWr
z8(p0%j`X0ICP7TpdO7WkC?j>WwPnxT{PJ`JPEJ<=W5Y>u$|rHM*q=?rjNCe4YuVMn
zsdirs1r6z_iw)XIq?7<>Ug-35>)4nkXdMBj;1NznMpEQ8UWvY`CnGJ>hHC2nwYxfT
zx*XMkm|CA6&ebSub8~2D6ehhx>b|=w5q+rTcCyf!cJ=n?fHvCZNS674e4}(ycIBZf
zLlv#UrmBj<z^?(Mmo}T-_Lm#2Vcy|_#y^HMxUJiCkc}?Hb*P(TV`GONs8Md4&HU2R
z#FDRHjl4aNW-bJ`ul`KNL`Ah54m+(QL1BD&xM<)-<}`CySYAE^cfdpUg)1MYm7Mqe
z><qlMs#<z`>8W@K<P!Ky_CF+vKj7qi;up|Ph(_oRwug2q*xA||va937r8$*I@Dub0
z5wd8iDJ$=B2S<k8*I_m5odaD9lXi*ekHf9)8|e6wrvZNUXP~cGj&wq?E^aoq2kczd
zS{!kB!RX*b+3=r(ck<GVpQ;UYk&b$%$;wDb;8<!(cg1qEFmpLqmL1gAxHsOT9HL*f
zv56=OvWHcwG2pXt*9X*nzR~V(a=EY%0w-`eP3tAQyKKY?m@osFfZ^LfX<z+IOn4u6
z@b6e!>a&~v)GbN?H`n#+#Ru!d9G+P&lIHfh@~mL$eIVGEHiM>I>||nQc1uEi%sKCZ
zhX-c3h3>@L*QQ$ai^M{XZy`d-`DhrXw`M=2+u7MWX7B`ci24H;Nmsrg*DAfvDYjkd
zi{j=^VxjHhFRDp``?$D|*w+k;hwrzyv-ixgXfAdokrUls;c!|`94%YuN}BAw6`bI<
zQI35>Hi1Xbtig5Teq<Y*Vpj8C-C1|~-+ij@&sG7IJ8s?eRd8ID+0<RQOo@5ULtT2H
zO+hH~mV$`ZJswU&VEa%uG~(mq+fKe?8+OO>n*EeY31=KlY*o?ge6Xs}nD4d6-rUge
zJX%dh=VV*d{;ZGt{ym|Eg&DerJj-I`ckPRC9E6w{%^DLEhmZ%W#o)R|oZe`rlCJJG
z#8{P!%~btM5KYrFGCJDYUctsQpl3^JYHqiF<h(I_uU-fy)S{x|nDtrn9S#l>HQ&E|
zvnwhqi_0<>6>SEAItK^G+<A5#A#W)u27qKW@mFuQf_2t$Xw!LN0ZiIb8d8DDC0?gC
z)j>FjL$kRy)G5}_rQP>)UqloX@B_cdSbqB(f{C6!>cwhP;Op>kz3bP(8q}R6Fk0+R
zgxDo#wS`9tC;bCjT9eUICZI(iLhtZ-Z8zD`nh<#k)bP1j>(tm^08~X(`q3c|50B(y
zl7rt{TeGd3d=8EeJ;A%AqA~zt$oPT$Mdj^F@rK6Mf-VzCTDw9J_YDsAuT9i201pFI
zi*@rRX^msgXRhI}=e6ejdqU{l?PaUMR0kO+8MUXZf9U72E}C1cJf)fusB)RT`tw(B
z0-sB*zrUEVF^%sxR}^yC_v9enI^`-cPg_gNMYOajiwj=Z7Lb1KzaJT?Jo7J<@pD^G
zSQ;DGI<Mv9XHq@5?>3m1C=<`^h4d53*KQV(0EIQ!tL2rnFcE|Kr%TgIM|3p)tc5Zj
z1~W|(WaE`*u3Qu#6F@v#6tvwn?r!=l?ooZrI4c(iHy&QtQ)$VPl7e3Y0~2-LlTci?
z9}}r-1yz+1=py~FwLOoye)WSIceXLW5kV~A%%~Iy@@GEFAvvH2QBvlVeDTZvb~WjN
zl(;GuLN<}-77-CczIO;Qo9pH^l8}A2+1Za;RYyn?#Ji`d1}&j0qh&1qdR5Nr9)pAj
z8*RrizP_^rHa0ZD>MGAmJy%q!UH-vyaCq{uWwpXVr^eL{ROUtc@wdZsn1$6CRn(sN
z1r}mQi3NM_uL8G7G4uV8#dxepb%%vZ7tR6nZ+)_AFf%eSG;jU$h0Dv!<{@uGFwdQP
zP(J2=d<-imEiLWzsl|SRTb!n~u}T`g?@Ml<9Oj~-QBCF>HzGg1yXLX%wvJTS4EQpu
zSxhWSj0D&0zdRaSO?|X-6EzN4VF9$E<m3+>>7>+;8ps#7zQxN2xH@6He7UeT!t}gg
zD%jX@rKhH(gi}Bu*X(m>US4pp#giv$8X-EJudQAzG`9`8HU~u&?XWQC<((eysVXS&
zy6>;T+A#OKz)khDJRPup`ijNna9&5CC<K=4Uyh7CXP<oM6d4#v3*I#u6-z~XeKZE9
zTq6HV?`Bp;Ho<TG{Y>}nVIo#ZS1(`2nR`NvjF3wb1OS$2Yo3<4S6KAyKG`^bQj_4)
zuja0AJzK+lwD!^>#N3w_5q_TPc3}9Dm$yx4){+bk)~WR{vox#*5Sa&({lNoeRVBRL
zgUB^>2f0%1qrmo@?A&4(Te=_JF-fo+)I9s?fFs$5R5zoOpD|?h_xC9&&%cj}YOhED
z=#I&{CX-AqoCpMfg!abyTrLGP6UvaZ&-M27pYD}Sg8c=)-wKCTXZ|yvx!=S0Y?g<%
zs@!G$45~rR`VB);JZW^y=X}5M5IAklwuT|zeRu~xG`aHD7JgUT#eFM;>&d>1h=?!1
z&hCl>B=q8OUGC(k)RaVGa_`N0j4O@s$#iYsz&^=M-K{ToYN5e-srLZ`!_Ho|Dq^$s
z^-cgT8lai@+?*fcKGe9z;vj%s1X?gDjoU)V-fCtSxiF~pqN8Fk>Z3Yh#NXZlufT(b
zN@B;Bo|L-TS=3H?w?FQ!{nt}U>2m1VupO|09u6)z0r7y8zo4!t3m>8VC2hocdAC_}
zuq$r|@PzE75uKucW(A7!-(_GXLI7nGZm@gjk#XQaoxW*iVUe#{v!EdTqcfhHc(?9&
zH2`l3N?e#Of+gxL9yxG6fh=+<J<;+emrlLSI$7sR@09(X;~oGwz~6gjay0&-=CI=o
zAIZ;wOZb4s65-YLTt?YyLAq{<!x-`CU7fdx2&Nv|-oZtvz}v=C%Tqx?VKt?djEpR)
zsQBpIw{MtOo4k7V_7RC%CWyP7pOeECL@(rjDJMo~*SfO3_(2?7O&A@mnP;r1-=JgX
z`!o;w2$QKy2WaxvF*zAYxK=FU)U;Ln#kB$Ta6@V5)vLOYhtUIEx5#_9He+I9c7A`W
zsjg12u@uwQ{aN8)(%2|`pCAY(BHFJ1b?C$|y!{KH%zmakI$EVEVVDo3RR=pYZaXbw
zZaZ^h19AUgR6#USby?@NhLWn+IxeNlC0)FF=R9Jaj12SQ70nV>b}G7eaGe_d{#jb|
z+Bc<fn8U=wq$^`i8O3kRDJVhzKbSQF5Bz^}0vSK6d{_zKisBdZ(3z<Q6+MC#EJh(>
zcsQF$q-53`zlXA8h(1<WsGjUBc3n6=QQmev#lpR75kb(nySwYkuPyI`PL(1$#XMJ+
zZ9;UDK>A?IZHv;WrNnGHnZk9Zy6pl8&jGOJ?!5M~P8TCl7RzUAme%WVlY)xpH?pD_
zsttSG;mBczyLU|ip>=&)PdxMq-$UM+K=^(CrL}bv-FP5!?_O1xGf1}IrOMMKxqtRy
z`8mk_F7hzsLu_nCWmPv0-etV2Zbvh(zsiUsu3?J;+ALco=deK=w9rdtX4$H_{NAWB
zzraYCy%J^yU;7K3i;LtR@O6KufJGdzbOSJQu{)lKcnUxf99GA>-6`)YUnnap%j?#E
zwHTz4jr&CtM7^q`DbvZjv#>T?%)rUR!%NM3m+n&coeJBY$Z{TUp~=aCQvj;vWTy{)
z*2fNcszOBJdAysOmxYUW`6}L3y&^RkZrz*MG>X#nguw@^BLa54a{w}YrlR~@LtWvY
z9ahBLUcxHIVoz;<4)>+6J#?|o)Nh}QW3}tuvlxU@E1Fqr_4gE&cT%$3Tgoe_RBCFe
z7+AtHTW)MM3t3-au*e=Rrc}vQ6SRs8i>r+DAl*JfW_{0MIpaXIgeAFd9|p4Y(#j?{
zPB&Z_*4!CwWx0U4z38KBE$FV&^gW8f!^1cuJ2UdY=vCg_-&l8%$No796E!D5buuwB
z`mRXU+_by9{kg}q=sm4mQh9mBbdr{)Rua-PS(aoo7&wMOog=Z4Mqt4K+Fkc-5;eHX
zr<W*NIsF<UIxz?~I{aK+UzD!=q@ZLxQ@m@})O^|GO~}(OY#40Csv~6)>H5%{kdu=W
zm8}X}K)kx-yR_lgge%|&(O+=e=U$*uiTO$BoZ;d!n^6@F4Ti;SyV;h-#YHTv3lr6D
z<DKKA+~=kXG7i5eglYoAp)T)~nvrK{An!))^4&WO?JfD;<6{%G-e`>5UK%Wf!9c;G
z2nv>0)O@?~Frz@(9}m5>6v+bx1XT=+PQ&2-zu}4&9wg1*dd$|A`====5?3Gc@$zB%
z{24-+(s&KWLRXirs*{%B5|lrAd3k3vvIQsDV<PH-nlbWRLBZhTf)nsB+E8bxQB%_=
zDypz2W-R5aD)+Dx85kOBYG~N*X$+7Nt7lnxaj?$h^-4ljt~*<9BV7(f2;f07hxzYD
z9*<=+MDE50%YK~cG-PiS5g}y!fCeM>+Ng_ZxaMC?cUw|`vJ>_NZXYkx_277}ZQu+_
z*s;A9E-WkrbGWpGEKR&KID5c5F=#VZvN84T)6_L6NQ){F2)ZP>`!EMimd{Afm*X<I
z=diJGwZ2=uLpUivOb*s?bu>FGD;+)Ev9K20sQ?YuQA_ywMtwLo%RCF-4S*7?$<|qP
zs^48D?@RCIaim+e@u!K9vLi*UAVoINJF64fll2r))%M5?p-{MVN~4{vWo2)xG)-qL
z-zEl2KBmv@!F=vSWsl4I`?&Bc;Q6p+eTIcgFbew{eeRf_q{dTKK^p@BEI>f5mERK#
za%;<<HXwFBSzuD-J}B)KX=%Mu{WZ*h+^M9n`GzQ^g1!9=JZU!pD)kReJ^Y!O8N{cY
zYH2EbGgDiP`sV(<fzI2+{$Ni^4JH!sLjEq+G}{m~GLcH{<<8gXAg`dt$MzaMl(0;w
zs5o+2!=y~k`22YW%3)&v>4xRyrCYae&t|WK4Wy`7QRB7-qM)$QZGROT;c>j1xzzN&
zzKCi+D>oZhAv5N-leKQX<OlD%zeq%OC-Ko*Uugwm=l;^DxQ0}G%J(H9U4-vrS}COS
z^73-I=2f625(b|&9_GGK9mPie4@>PdTXY@d<C_#m3P0|_WBHIxDEui{o}jk5HYwD!
z=K3!<Tlw`R$I43oi|S~5CnryW8;y&Dy2OMxomiLaG5Bd|n!$+;U}WvOUb7?3$Y1ku
zdo1qmCpkbk3A=@`T{*t$j)is6H#t8)KPANhjS@h#b$49e+-u&jpwnhdzK*>@$)Kdh
zP*hS>QdoGw#}{iq=DQU_kF^~r!<m_RbSy9$?vIMPqpZv%!vu6W&AgnPAHYfhhclw!
z|2iUP;wzcn)->5;i&B~p5g8an0yq%N-|gOf{xfVrZ{1zib%|Bv<=;i_v;%Od78mfo
zsL@KDIy$}j`j=+F$g#8S`7t)u&@iJt+n8zvO~0o@SO*&h#L-DHG4U}LdPWk_Aq5P)
zRptEUXmD`p6FaXA$bh4ZeX_m}wBsqQZ+ShpZCu!Z&?uCaUd6?Hh%q%EOa%Abi{s;G
z^!@GaN3ewB;Am?wL^$x5E2YbRXPlq=GPej{Aulh-`}b|qB6N(?cRr<hR`99}3}A0{
zudFO*XJ^eOeH~`-117I2fR>x&N6Nhf0#cSa3@)luQW)CbpZIsnPfw4{we6aWRj1A%
zDk3(v&~=IZNac+?k{HoDkv?D2^Y&S&rd^?(hnI|%l~qeXSY>6Up#k;q;RVD+|BEOT
z#)`K_K+>hV!8KM50Q3yKBVes->bgCrbHEIb9YLq#)!#h^`>PZ8-S3<p6{mY(=uqK<
zpLFVF&%gi|1%);vVX)t4_$(c<#H_KBdwYB1RR=uY)083-nYE>%&HCr+lQS~xW@+y5
zI(sbj&MPQj8Hrzt9|R&3Se7x*VGic%fCN5RhuIsyx58mD$&18p_jh<&V51iaKt%S^
zx`(*)FPqF~^}Fw{pS(j&n?<_6fB5hrSE8i>%Io>g<+CC-QA^8ayfRQv4feg7s|MWM
zWrXehJuP5w!MizsQ!$g1W?@>DNh(zlhzMnt%|Xx&oJqIFPESltP2HHN)hagcAA6%7
z_dHIiIz6IaxqbE~At5fnprRz#26C%l*A#(|`E<e3Z$0<0I;Ox|fsTNhI`!E9n3T^&
zW@|q$Kl?=g39*3jm6&WO0I#y&3-bGXFCfKxv;cLG(t5l{es|~Sn}|ORRZ~p#drqbh
zdFE!P!+a66NtPS_?MB0*hk(+p$Ug1(MZn+lv+YZH`L#wWE4AzKjd?fwXV^aH<*Dpf
zzP9m8$)ca~_TstG$a;ilZhqz#6(sM=%F8P)D5w|e_;@?~DJx0TMZ_JhR<kz_8-kt~
zKEkP+7ITSsdhF^aBv?Lxziq~QidheIfRWfTOrOi%39Wd`n2(H%s8PvjY5t8Uq+>L}
z7xh?mon3VJJ#k*%s;6wyi5;G?ar7sV1qx!=G|7r|h)>@Y=DEN3@7^MgH3sw_GtsSc
z$%wElr#&aIb6awqWOw({H5_8!xOkQ99AMLtd-}BZpBA&DOFt|Y@DhAl9oab&cqC1o
z7U%yEVat<sXd#3^2$hH|U!>YUHtLLHn{Bb4tYBafI@nAWAs6y^r}XCa>(_v-N$HG_
zd&>-n@*e{^8m;-6Ui-V5!P&RfXUa~*6cl=Z^z46?oQ#ApIxeEp8$Gd{WnQE&8Zs{a
zN{<sKHTCpNbGtvYb5x|?tj`wE31&%<Gl{+y)v+9hqEjw`UqVX@vylpf2qWcNQa|gS
z@t#91N^TMPdhR!p2-a#6NKuvK3nYmR)wOA@+nfeBPCFL-E^tAc^g+uzD?8KS4=1nX
zi>?0j+IS)<(<cp;-@kWU^1Qe34HGwBps!m_L;Pf7jSpz~F*}kJ4Z4(sAVa@z&7Mh+
z@hs~PXK}~(%rjpxdaUkz2<_x~CZTiUG=t%d0j;6raMj`js(CEznjQ6#HX9J!0--Lu
z_<R8`Q$Iy;^Rd}nhEXt)XSkqS&_Qfpx)D)n`(Zz&o>2=>3#aVE-};=_k`NJ@1=U7T
zxU93srwW^?B^b*{$jF$XTn7_1%EUr$C87jnoZa;GVIW{3CU!BbRcTJkg=cx0L|D+w
zW_xp2T2?kIJKGcb#%h$MWo2@2!e}|KCKvcQ_(bo|FBA-04rhmUFwmFBS#7Rk<BB^$
zN5sLn69*g5!O3Yh!yYDTdXyCvf%A&hENFv=RFNHnLb~1S<7=a3<b2kvU}cS!u|Bw_
z_Odb0?9NAe`tQk1@9xD&bLa=Ebv=7$i9F|Xi^Eu7G1GFD=4`1N=16DJspPc%S+ATu
z*p<YGl|K&~=^HI!Ya{($J)fxoq1oc8wx5S1Sa)%9f6n%q(Sg$WbKe%PzkE70GV<`j
zgS}U;Ud3};KW);dCM+&34V6u2(X2KF7a|xt#adbbU;Mn(QR2FU*4M9JC9k9c>xhVu
zkX@Zo!HTIZ(ZJ<v8Q$f~I(4YQS2W#k<uji_tG1%p_>}38I~!K3pn1)!>zqkT9G<7R
zWJ74)F0g(^yhpySsr^Z@r1`!x^vwtAZ=j+7xpYW+qj?^1=tU^%YaTV^dc<6MV`}&A
z%?>*(4h;6UHaFwT@+Ap)(pO=-qsE5cycxD%9;tM5cqIRGbGk7}&@(P7>NU%)ON;+r
zoN~PT2m2Y=-s_V!&l!5pn41Tgqo3=)CF3jAC|l1~&8ZK=yGGJ8G-SB_D|0$Ixw*L+
zHlVO|h!<cG))HxKY;-s(qSh&S`L88vXM7L<(Jhq8?^?4nva>sX98S$FV{E#e=BMW;
z4R5dAJ-YPz9AH>e71^$USvfN!qeP-mIVvhEB%T<ad_zM#EG%J`r9e)6Hg6k*fp{b%
zbJcsn?;2VDOmA~Az=WQCfhm}i10^NN?1_#=DN{TbyJ-@4lVKqHH(pEGtLXf9gTFQ~
z>KhyT8jD9~L@z%P?)gwS>*(k>$N8x!B0ViF6iX-XA`xPa-%Mi6VEE2|?BiE1`gSkQ
zvOVQ`Y&jn2!w|y)l~Z<wqmIYC@-xmiE54YRwL|n!lCQP|=38>yYGfU5kRPCDcb}i1
zw;nBeEtUe1D<OXU=v6yR;WSq{0jKRJN{iu=sf%471ssk!5pRikaq6rde;A^v4S)aP
z!}|{z4JxtI91b&0?MyG&?1PVqVk<ALr2+zR=PGC;t9(^p4AneeY}{SzanyV2evt8O
zxxLS1&UqBTZ_4UNP_X5?em$Rzd()YBBA(mEORH+%hFa05Q*ZX+d4#9uDHIpsPyO_W
zUrSP~UeY0e<z=SwsfP%C6v}OHIY~8t`S-FIv)a3;#G5CFY{}gn*fcuk@n`HwL;K9k
ztXU4Tm-3$)P@1p&$55iHqEz-<7m&ceO2H#9Ei6>8HXMdA+g2a>B(`#YH^r;Q>0g@`
zAhB>h+DsB_5^{gOe-QBQ-<U`eIX|zuEh%9#1EY;~{o-L5-7+mpM#=2&_y77u@4D@e
zS&9P|JiW`5i`?iw;4^t(o^#xzzkK=f=6mRQTp@k<wxb_8qnmpNFr_fkVK088!wt6e
z@i76l?>Q;0vIs<Ev*W>L{UEwi=3jMDMC-lPj-YrkDuQ+~@~a`G>cy~K0*J@<1!M*$
zR&>7F{cAY;8>8YMo~oRS{dXIX>}-Mg)wFvGmHcFlM$G~ddD5+-P7`(pI=YydlBdiK
zg^FE=UtPM%s=5*(c4KRiv`s%hflc%}BO~MP-c@}A^1=RoZQykwn*yO?qD*efmP_i0
zVb&*(j)+KzjWQc-$JIf1%-qy?9m0YHaTB9Lnwqd4kV1gd2mvARr<_FdX0)W8LO8sN
zig3Ixu?exT(9*VkAE|uvCGQpT90x}+RD<Upm06GP*xSGKz}Gi0xQ&lrY20a!<ovF~
zR-u%>y={-wJ6X8*DTf(Pt{aBryStp))EnJG=mheLN2MqiXXje4HQ3)n736mFCZDq<
zS|3xwuHw8fM%C8EQ`K?TrZxq<jmiA1JOBYf4L>`#dWVQeif3(Aw<kHxKTHT;E~R(N
zH`~zZCc)fn-HUo%R$MZ{gs8-;RA-Gk6^{w2fx7L>+wj=kJlHwGIzKnR&_3s6XQx%_
zTDU5Uk$k<%acP@^>FPltc=7$jf6Uldz>t-V>w%!{?`sp!?$api-6!XBe*d2p<?mRA
z<NS2We$H~4x>9T;iI<yOY)T}sIEh&|v-A6I1p0}IiT$Atyr2E9rS<)sp%eoHQYK>D
zefI;mkYbmLoW6cT{WlRh0-S)9GbaDyR(G2{O%tu_-g?MJ<?g=-3HEUyUUrM(N9rh5
z96s>iSJ)4wMii_MDTu2Q0UbeLvf52XB@O7w)j85s1a>@G&`V)7icYxL)Y=Nn!4HzD
zMQS;z9WgFN#TJ=Pz<EeF;YMuQ9{l+Jz3IS-!HpkNL<C{$Xm5V}0>Als;FI!;b9K7D
zs_h-^?cbtuveF(}>|1)Yc|%Y|*hkp;50pj*7D^c0T{z=jS~)x0g&VM3Wp)AToTM~$
z;FImGMl7=%tr<}Lv5d@fGLx;6-g*B)@~~RTKR$jmPRC1GpB|tXc()$z0=nBUaB}@N
zFk~nC(lC7F@BV1-RG14ed_G>+HWvR=`>zCudkoPX{Y?fh&Zw3?qJp`B-l*!B7$S|=
za1W--9Nc2FH(U{x+(D&&>VPJ+x!D%4HPks%i{uCdGysl{j$CJ<%!96gUswvi_Uo4~
zDb^JRywNwKSnhIC>S-C)vN?~rB9p%t_4aNT*-O6)KLQfMm+H^e85uvbU2G7)e<u6B
zOvXU{XaK5;K$Q~~g2BSVpyCu70O#sIH%o0UMimtozotxn);puqN_7QA87?oQtnAwX
zjY64&M)%#{7Z8A;p-H1LGuvo)d^5Wy@O8<}K>0yG>huJ?)q2E2!a6*90<0>Tbe~Bx
zWL8#=z5`D=&-%JIA4YIpB8(p=hnlErw#Kd@asaWJmM=)TDjA54UE_YBprVqdnCW}*
zYG_PV6Uos|hBzA@L*V-Lr>VWLjpX4&&tJ^Ef8Tgv0f_cEd1%<{_zDXfkO?AZE|{@f
z@;O1GLmZpc@w1I~r5f6NorCP)VunQ7e^*I}qFTlP7LQqj7I7?bERv8Gh57RD0&}8V
zX`Ja_%7@y9p6ci%wax;BJpRK6l`VPz7)joZ3@%_{Aafs;3;?Gl0taVjd|ZI_N|pOY
zD1b&3W^<{9f$Q*lW(EfV&F{1{W2@e!=Tt<~YWGc$pv&fQS~heK#)T)my*|tEhtHob
zJ)`d&oa}CDI*%9=;BUG*v%S4NGBOgLBIdNwgNe8j@|KY)M{wT5JfXdufh`KIIa%he
z{K@KzJ_lwSTAJg)d8>@81Vm~_3`@wBkV?lj=md#}+#&@Mjv6B~6N9p%;_}ept9U?4
zdK^D_J_zkmAbNoUAWHdtz$qs`*FKxN7BJf!7CT2PjdqB@?Pm((L)jD__P9a#=t7?U
zcmXMJQ23s=mXw?$@kh8l{ruzu3>|v>4i70r{AZkFxyG@)e@fd`^!D{KF)~4U1~lgw
zRhuJg%fVsm)x#}dwB|OzP4(?2EOU<b_Sjc&fu9U7vIobDO@8NjovemRhC4e$BvT=v
zWnJXO!`f2`9+P{HHqDDQhNmkMnqzMiQsBcw8AwJpzB#%AJ^sc3Je{f+PJISEequj5
zI}6u`t_yVx^iOD05p20u2jG!OAMB%$DxYA!s9kNlI;6l}K`=O7CksjsmzAYd;KrrO
zXF862N^S(8@(<stTjYIF>Nyjh$4X|uB+2BQ50hlmIezbs(3o)SUcItPnuy`krOT69
zrKh;6Y;Nx5WgR?nH%bzZjDKozQ3(3SRW8;Y#*6V~)=Go&CiHA(&*CD39rPYkc!kkZ
z-*@{tMbRqzkpSDTPF#FeD_9rxOzW#t$Aa6=y`?K+Zf3*&dn*{9JWp_gW0)==aO3-u
zJ+Zv)rgCp{cPB!9vwL5t&S#C!8E94vs?6y%a`!B(VDb`}xm0=eHXQ*S9o_q#jtr((
zlF)XhIXpg&{qTVxih!K=m`}ayp8=lNhUJtJ9=i1oPI`Mkdj=XBIGt^d)L3L;CVFPP
zUNcYbLqyKe(Vl3!Tw>fqr;(q@jW7*XRg-^|(ubkDfx^5(eKENGvphs9?Y4FOcwS|^
z2kl-+)JpC%g(!-{YzvWd=jIs+;)^|327Eq!0^+4U{_1i+!(k!%|596wmV;IEwZBXE
z0nizkty9X|+I26?ObwqH=Sz~Y+`DJ66Kw@9xx7;6?vGkU%gj2J#KZ)|1a~y$-_@CZ
zJBRa{-R=AP1tFt^ckiaA<{YPVpMjEcxY%UVCGr}L7oW{!?OK1;L7sqn^NfE)L_|%M
zGv=d5x}SpMZ-+=4WP5n+93`k<XIj?r+gBW`x&O>uK`$9Vv3>xGRa^qq$^08z==p7`
zUpGoMVq?Qg67nuJ-HI5d%7vH<Zcg@*O3$kEjlI1SWyiFiG_ImdU;Gc9c9-{#)t@jg
zvK|{-Eqg0Fl=omGqotyKmbO9dy&n&GM^4vWkmW<~eijtYwoKp1H|hQuakfZ5p^vg2
zFQUGC*S6_R_*W+<*c>%+SHb=8AJ|P)4s!#pEY09}l?x%`bLnJ+zeIN;Z(ncmQ^Rj{
ze*xYk=ad1l+-9UGs&g7iMjAq1O7)8(6h}k!Yx}n4zhJeeGmmdW^hcoDTXe$S0a@K4
zG7jpiyraFr-rnB#b5!(+13ZfIQc_a36`%1)&0p}SXvyR*`BUhr28DTB>l+^f#|N}d
zdn>B&rBWz;y%@7H-|w#C%A@Y@ey|~Hx=qvN@9$5Pd4Fx>Ym7kT#>~t^A$tsz{e3)_
zZU-pj0cVW(@uRc*<B13zep7SHDaxJmquXhV%-WAp1+!lui^8lgFH8~O=bzO#Fe=+r
zR8qPi{5XbL3ley^x&PSt!kR8kjEC(xE1{VaT-=dYbAq$I{gFSgudfda0c2UFG?@zs
zqnq;~@uj*xYm-$(br#6x55inpy!fOw9?g;$mF}N<jsVz_%VyLW){KUx04fek;fhNq
zMKgtExsDlc3fvD?aj)QfWOFc9S0Da{c>=YnPPOY}uJcH{_0_SX9hdcqorTVo_wNmn
z(RZc`)U=dyb<^)fN^NC+LT+Uz*LroYC+<)ZRlDz$W>p5U?ni{&#|eVa4QHTZUkD}y
zW^~kh0-KQnGNHiI=`yE2Pvhg`lgmf1fTS~Z(*KNP4<e<qRr06J&1tBqX2O~N&iWCh
zvDDI9i)M;XJNfZ*VBiIM{d9et4+C1I))OUmY57S-ML}v~w%c>q9?i09sxMxQ!2Eg;
zxeY?T;NajY?+r3_Ei$r?=F7|Bm(efgO!jjwAqT!hM|0)R*oa{3X?bhce9la*a2n-$
z=nGWH<8LjrtD6_J`#9<hA5tX*f8=v<PRpO{(yB_mD<dJ>-gY=dSs@t|jOOw78uz~n
zsOaN83urQ6A)qI*xIT`kf2M!#5wyqj$%O<7oNOmQ3V5L~6aqdsf8b#gM1f4NJuMN^
z2R8e+N57in6Ex1uCe8AV`T?&z*%#}Tu&{`5qmakb$*%?h3B0$H-OEfTq=xG9v@1TR
z=hvA`qMj?Pg-@nzwYFZodh5r6t*rp!k)7REL)(#`n8)67|5u@WxWrZIqrT^74wviI
zB*?JiR}g?E5VyG@ZN)p8bTTk9z3cq>!|&c1x3{`~?s8|Kik~oR5|E>j$zz?)=a+Tf
zpjA0fLll42fph4;XkVXaTP#JcOT7TG7IHs6p9vCSWWBsMe>jXV`TGX(5O+uu!PLp=
zCVW>O{l{PApuCdmsLST3&`{hEF$X)lDD=8olS@NS`%;WkU5_be-o-%(O5xzZTmo_w
za_A=<Y<;W}7|UY<?g!9rAEt1Cea|t`4EpvWizc!KzXi#kl^dk!_eiKN?Q|t6lVFQw
z+kJPs-@^MFRfKB3&TIdGfCwRKidNq1lMTVYep!!KZH!dOQ4%HzI^lTB5dHC<kyDF*
z3LSk^kLUI$>GY*0qEG+-Nxc8hL&%Nhqeoo|R+swA%z~W6M=^Z;aqqB?wfBc~=^}h`
zwH6<X0I13T%I%)qbfS++ayoT#=$DXv`g@|dEGuiIjC<jC;f?3dpEFfHk&;@0f46Zw
z_XDjhJRaiGz`xl2k?){2(h)*H>&OKv-8u%+0+W;VfbtIK(lEOK9i?*Cg8<gp?yv(;
zuZy05Bq-ze?^#+okY4vx>Z6@1YHy%FXo5=$E`K<x+{cH-4hJ7U*1W(#1UEnQo}S6%
zFp*2-tuh!~M`2z>Jeq5G4Wmo*7i*AR_2OWO!)as};!i*Fx_F(0Cc4Uso?Q;^Q&V`x
zUs*2U;6mu#hlDF2o}R{`4OYlC0Nz?@;=@xG^S8MT5Lik%>@yTpxSbtDz0;^NxEHd!
zHfmE}|9FylP{=2;cqcTm3N&xE`!P<CwCeh~u&{j@*zQ8kmaSb{*6ZurQp6C(IB|;g
z8JBb)@6snlq3+#>BaUOG2Cd-+tuc%mEz=n&mdY&{NbHmG9G>)~D5cA!%hOqDqloN$
z-{F!zOgNc67u;+f*JfA*$8ffHR{0XG*BFpLS)O^bljFey0bADP+eVf>yEzq_pg9NJ
z^mw7epubE!?4_Ph2k@MYZQR$xY~DAIx-Y}88KDvrCFjk`&qp&QhuM13HI7wlX}o?9
zZTlxb;!c^Ufl@XMiHR3DHio`dp?zM$pyuq0qPzc9Sx2dQ$fy1jsJ;;OOT~+K?%at-
znSJ{#3U4{aQ){OqA2j4MXxWbsIqYV>*yMvFIV$Rw=iKk%oy980^>b$-KR-7&eFN||
zhO6?I+O`va^)HmnA18g&Em_gkGaD~7`}{JwFge`rhp)SfwRXDFQnbndVsn2lJAN>4
z1`RC&pc*24GMgOjo!!GD9A~>bsGCK#NE1-S&rh6dTg&%YxTe-1)sxH8GEQG#|5${C
z8FKedE1h(x)<?ef!H_*t7p;;F<2(S(?(C0`rUm{t=hf7y5M8M`8R#v|?BarFU@!}z
zeh`s`EHd7MjM2O_8Wb(aj3sWgKoIHpsC%4~_)0^Au$9$X+8}^YiHL4u*29t~o5+8(
zIW02V60$L<lZ@dG6#ziyuU)zzq>W|iG-Xbf@?0S&KK^!6w>L<S(Vv(9%X|K6RcKFe
zqO&7%L#dA|e8oK_EsyXzoBPH`fxh$?>0fBDE?$^j`fJcFFZZFP72nl6JhsC#%@p6h
zwb0POuI?g&F!+H<S9K#+ibt9ZK>jxR@;A^4cz{%dksg!%M8al)JeGJXnSeX0@v1J}
zecp$>uSK75hCna#D>US!Vpvoa^}W8HhYnZUPdy<t|H9<zr68H^wz;6ch=13K4j$Jo
z2k=zBc=2(((mUm@ri={qx`7Xk%x}tAXvXit^G5gyCqIR@g&&lh;7ZCq($dtbD+h`M
z>^5K&mCK+)MG)jCjJSs5$8G=;Ecm3OBd9fJWD#|+XPV4jESFPS_ijVOE6`kcxNCWM
zUDZzOF3AkvSs)~esh0D`L!^kE=6qMUijAXLV?IB$H4D{_MtYw2<rlZ3=l?(1*T<uG
zRAscZG{g`3gP$Qj{mg(NFCz;Brcc8wB#73Z4_*Or<2m0~sl<f0Z!<_e`zw;!qju@A
z@6B~zEIeQ?lXD2J4}XpC&JWiTPL_up(|0lPE+B?Q*xK4A{7(?<_v|^0df*1fO8$UU
zA-zT~efHSv*Y#kJVHmzU6nNwEy?c$6lps_}5^^s!Imm-s^Un#W#h#;Jrr6oseAD#h
ziPU8&xCl>A4r)LrH<~Ll<nzy}Ok0EdZ~jM>igy*oFS9gJEbildEkrgrDD<L_a!1VK
z!h%=TB_Ko7(UmL3FsC**v*^@va<Na8T6O%?YXpr&eol_1nV!15e1TD4TGleXl2Tu4
zM>HUP&-sKGZv7Fu3aij#3W~#$rLm*p!Z5N9g_tDoM^H{?I3D*64MFD$*(gvfYVTQP
zLr1p?3l$W-UW2YFj{`Fu=YDUs-=GCSPC$2DYDzXHB3(A&Z7__FGO=*Qay1U5K*E5~
zrDsdQg}{fK+A1nsb~99A*5imzcZIwc_tz$C`Ri9k3av)LTyX}eUW0~N^5e$<9{roV
z^SR<=sGz-L&#Si2pD7V<f<m<`O|M6ZUB<@F&C2WT?{~LFdjVbEMzhZAWI*O0w9<YB
z(o9yl4Grh6G$KA_Wc-eKV6ka;L7YCy>+l_l9|huJ+-A|{Qm#@+-Z7<{x}24j4O!no
zL6^4@V1&q`s05=pHuJd*2#KBw{2w8!B#2)D+yGh0oB4ihAj$09-77BqnyN?##agw)
z31Msp%w?OMGVXRUGm8XV)%dFan$akz%*E5Qv7*@I(#k(5q;30pK~`kaomgerJ_B4#
z8K%x*hXphHidgkn$t(T|pt;4XTQ-Ii>xo*Y+o=AQ7US=~%)Ar$R)~n36!buMyE*NP
z<yZSSLroJ8u-1zDtgieM;1s~Qe7QY~CMURHG@}2jM?YK+N=^^<9(mWnV$Ewn9o&o)
z{00am$-Smq-5*K$oT0&qKOmZ+#x6vay>eNIYNf+y-pJ4QK{9UGKPksOPLU_OYYf+=
zT4ab>2rhx#r%jkVS<`T~2%N#*x)78M9JYyKcb6B><L*q}z3i#8YjiOY=IR@+A3RaJ
z!#^n+H>X(^zWG8lG-my1nGFDaOG>_;=RFtE{-G82cfdG6OhHH`ICuat3TePQ0b%5}
z&FiBlQJzmRF)@YEU=f<FJU-?yk)1X$g31D>#<E>q&qtn1U<ZPWsFC&UkNKx9U*R?;
z`Ietq>omInBVnY6Nkl(PMgE@lT;;rW4CgX#nFLYpwI&C>Zis6Jw-67H?Eb!rl5#F|
zvkJVujg2qrC7k78y}j*MJH57MWolaDaI&9PbpXrkRdT_Z<^B$kQzFfGHHtjcA8UL4
z`i6N=EF>Z+($?z|LeU3=?XG)Eqw=a{4y~bh<h|{4L7L^P9QXR*nWZFETu@+pGLTE`
ze_giw(hBAKVBzGyYH|65Smf-4n$VmEEk6vz!x*)zcQyPYKn00(+sk@N)lkQUcbzFn
z;V%KpjWg9p;m|sSh!zYVnC#WQ*xH6Y(snTQt{X;O=WUD^2!szp;>qJ@3&VXTI<c}h
zj`(A5vl72Ox=g61p$^H6R^!#`vg!BY4`9%PIv%xw80CEJ>ahxgs%P}n43DV@aL%7g
zetMVS()@u@f<$Ecm$Wo}Z=K`)?QKX2&Y{bHr^84AWBN#a#9adHtsl_zV^Ga^Be>zP
zvvBodVh=I`0}~4(qan^2wO<*>?sU8h9y}0c;31$|gYPppKO0Vakmv4DV<Tq1>{ca8
z3%uNDyC0{&zBE;iRHg;a8yOu&XoKCN=~wlqyLC%COu1I<f?sB>RT=Q%unm2CSFX5K
zZx1}UN5?bQRG0$l(5NVWqnk$jb`5@k1fQPxy=GP~$*`I9(yDa6fN;V4n=~_RUit&;
z^+;WaMnn9OIqkNyzdC%A<W53D#GPvdev9|%=~f4F<7;al>-wO-1kQFyO0_?Kt*54L
zrf#bCWH(u)uR48oyf~gQouE-#TBfhBS0(?s8wy#09nz{=d%XbyU^CSi&q7$6BqGbK
zMiteS&8&?2KkJJMhl9ASFOwh9B>2K(b*RVB&lQ4&q442SM`Hd%mMo&4HLxn66-M!=
z9wyQ6qeICUHcF2{ubQV*1rZgcj!XLDShT$HUVo4Oy@@P_@l>-q)Q#d{iKfS4lBx8}
zmXEbx*1i-(m}XeOtQixtF=9slN~2g9sm8Ns1SEH=4Mz9DE^r3EQ2Wss0OHe=966UV
zyj$d;^MlFFUm*Vm@rU~$VHa@sn2`QGkb45gF}UFY;tyTdLerjC>er_kX)J#I?qOn3
zp)4#Z8L2XHp$<#W%k$Em)b^-q|2x-TQbpL_-o6Z+b21+r3=5%my0m20Nz^S(O%26O
zMuxnU_+<n<`Vadu!lkLfc`~KE8MgptQZ`QZ2vXYFog?L_^CGCGz({7za-T3cLpJkm
z0>I1N0_U3UH)N7{nDt~WW^9LN3NAKe)Aez)o=4C`zLt{&TJ{d$#Fpvl={~H_?#zfT
zzcuUIG>X3hOZgl1VlJBzH5HYMh}jmow(mbKVdFk@KQ4i|TYLTYGZ^Y(p#FkkLva!N
zR{rY?uhY`fd;@T`Dov9Q-b=9Ikw+X|Ox6dp#^$E_Xqh!O-epe|(#=$@JRu9DYpbP(
zb_RPvH$j;U`Lo#uMlg#9%@)YUVKx51Na)1~mTckXh$9L;t<KYTIV?AD2MixJ{W<=j
z3||Qw@fnxw>C-nt&Z`-aWc?$UAST#S^CMBkT?#=8Mk?3tyzw0(#!r1qhWimWj%$B+
zz=*nN)B~PzaE1|g!=Ezi)WTdCf}VLy<=xyw$=W|(MqIgaC5Xg^7F0~Iuw(eR@$G;p
zj=CdyZQNUTAcyx3mw0H0D6{4+6?4~Q_d%Ic-NlO+SC*GBAYPiEUtlmt^A-s)M9R<3
z%=CUv4*F|kv4+jHfd~H}ERPjF;<vcDc>z+x4V}!)%tU|0)_|V^^wBWU{*lNn*JUdQ
zJ`c}X5cXIM=5)2S9gg}fZp~f@-+^9?3e_=_cDXHhi|kb_bVRkZYT$v~Hn{;!T#Uyr
z?E_eIvTIJ$LT(d^iIv(+e#?s_Eirr{Mi=upLGAUU5Gi4y^9XJiNTA!sGh3Qu4<#nK
zHSAPc1u?&<DAp$>bYd7jv42jfmR0$R)Q|t1n<}6;^Mx96y_5I=Vk7Tm%$`Ks^sQ%l
zHI}26!{pgEy*JrC8%>({gNHm1k(K)~1yi~jRkGXU<~RPx0iwHxBPx94Ss}{F5`6{Y
zA+KH9l=7p*IP-cz{xme#pHmWYcgv;ph@p@>&6mAPB(&JyhRDFsnOwl*FvgvT;DL)x
z^UATyQNRRxakxemf<)-u<_{IT?|^T?ifL^uA%TVOZKyTY<qn9O`TCji&jHE~jwU8C
zkO?7?s#xnoe*5;y@89PUAUh7z*FQu8Kk)NsTrpCfy9}+3eZ_D-LrF(%%RtQ2{ryLi
z3%iY0o!-V2On?!OK;3Hl09TZ>>EOrw0Y>=K_X!Dy>*H1V3pv@@5D9kPXEle>iZrpb
z=$sxV+(H)SSqy+h5>{|Bq88t)JCXhbgf3y^&T-}Oa@O*pNw*OQb<Ga7%D}ZqYGpa+
zWEtKz-25*nzj5onm3_|U>QW?J2-Nt2@^_K%qjF6?9YAJpxn@u?lxCR|k@2WTR%tuE
zoy*&Mdslb&pkd52wXkk&YWg822@4T`CrTRh@Dc*<2}qpO62U~QkJk<6in~;&@2>VU
zETAyQ$}0Y(i2VYaNd@V*RaJZ;VwgVb0xxEq{wDDR6r+ZMx`L8cWB&Bw;ykzlKua;x
z{AUGmFP7cT&Nd;-^ETt+&dyFe(&N+g@Ox-NkIY|PT9sh?ZWan-VP+l}7?{qmH@6Ta
z&E+#+GL3mp;60t>^z|#`Nwy=aQ9^deLz@+^)6ZgH2futd*xhA=b8WBF;C*rS3S>~T
zU4cNlz+lhSg$1z%`$<U5As`~k&dy{bep~<P)6HyE44?3D1+T|~>RSJ!FMiOE9oha?
zv*7pPmH{Z~>Xj_5ERzJ?a1gM&*ZI*?M|~F)AtGV6`+*Hc_eh2hzi^_WqCi#UXOr%&
z!Wd}7scJdMtTmZl(?~n%vOZP{nLD{b*yo58v<$~pzLh`bQo*qhjaG@~f@0owvmOO~
zdzGd}7UVleO`2_yhFWPJwT>RW|4l=&j&C|zm@Uft__45%t@fORoE!rKQ?uOGwGk=V
z-=Ok9fc54~0@}0tDEd#S<XgU?eyp)SKy&F+U+=UsIM_c>q$Z7+vgLuvxA<poJiN=P
zJ!X+LHSNrU@apNQfFu;MrgI*iM((%m><Z5%f6A+6W}`?>39$;+?D|sx^xf=^zp1z{
zA+)QXuJx6Ll}2-kNA-EvBQE;xp3knS5qV*d4St6wUGcQ(VK-Vr&<%HdAUomJB-sms
zowl*+#Dv)c#5FWB7D6XkS0R2pI@CFyb4vc%uW~yO*Ybh&XFDUq*RXZ64f(F!Foa}p
zC-6pVZQ-p*8kw^=?Q**#S`)$ir*9kFY|(=LdJwZx;Z{MY5<Quh9m>T;F65N~)tqE1
z42b4UXm;J$*#(WK)=zo%!*giy@yMxmCd!9+1nbgDOSxtn#kI7wc&S&%YEMB43xiUy
z;e_|akR~Q<YEj?2mm(7IIgE_&%sh3o6zAg*OvT*7ci?{t4Tsr_^xMm!vIc3<x%wa@
zsk$K|il$<XvV>K+3r9!xPfSI%{w2_L;@?#Kn;bTB@5$L3c834s4-5hMX(Eh@$NEu5
z<~ScsO$`+_HQ(rY2a`f~1UIw`yZa-jKu|u!hKSP<%q*Chn!Xz!>YFnE^~-~~06GD%
zl|=Al-N8r-qvi5^>f5g0wY$^t{l|N)u4&tcYK}uyXop9cx!K?jWMN>5{6o`;L-r=>
z6(<9SRCq#>NCW<r6wkG9zs>@~IBc*GL@1sxa>w&$dx0w?{O7@G?^bNr_=5N9wcuvO
zOoFf}#rvGzri<B5p7*3_EOZRMpq$p8BUx|jX<2ermH*!9EQRdk;gnzfA#x?#M;=G+
zB;vT_WPDy@MK>9!^tYhqoA)z-S;O%aa<Cm2s#%(v?)KN`@&t`a*;UkDZ+_j`-Qlof
z>*DBG>A7O8>V5)26&`p{rg_ZFNr2pv7*w{I&u}iBWtHsMZ)<9e!;})V7d2+57ZG&-
znC5-DRvR7_)jq52{I#sRqXXKXp8LD^5#T1N_D0JNtjv#f(Ta@y^l`Zp_~exyajo@q
z0k5n`g8RYJWds5JYn{dzC1Ee^2^cS(#l7K@sg*dYJBt|~gXI=F0MU}r@Mz|9S#9g+
zfR+Q~fFxS-L0hE~ZikmIA1mnf5Q%1HXB*(>s8F$tNPnF1T-aSoyNuaD+^ko8kJ4Cz
z^$LR6@pxykyInI+^x?Cg+Qj}*(eHmREkV9_RCKht1=zor2?+?^^;4fEnj7ydi@m)g
ziXkZ~>ZT+|cu6o`WPf@TlO1t|jGgMigVc<Sj6i)1L=ihXI!{f1^6s-7%ctfU`9%Fe
zkdj+Bl!TVlqWbsx1m4Vgy<(H&BeNCL$tusqrodSUyyYbGnht<5=rM(!4$YmM`QP$G
zt|i)#88l~L8a!t%$m}q52S?s{ZTMsu9reMSWZ7`vT;VsSPu~8%p5FAzAbP}+@Z`8Z
z8yn&i*39cndm^Z%2%*(^U0Xv#BfCld#p$w>o*qSHGk*WTIgsQ?OG^Ie=(ymMt(p%4
zsR(6Gi~T#As!%y1>J^l4;+$vSdEog(YQPvG&PO*W9V-@mN#Z;6%vP_@-%JX_i0n^2
zQ(Tj%I@g~Z6`L3v8(R$I&JI3<G0(5mEl&t5|5sFda09C}O3B#IZuX=T;}S&z=Cy{!
zoh@t)0Jy3#E)?+Er+q4VNBIp9lWU`+66t+J%-SXH2VIb%ea~Pvib0i0qw-4zwHJ?9
zgC7VcFJEY|aTCe?kM!DG;Pqp!Bvp@`^FFYp=$6dOG>eoVif@cQqNvSg#1c-T6&d&y
z7e|7t@sXx|p@!WlkXrw!RzX$gKs5W(iM_pIFQ{S=1x_cSZAUwXD)l#DjtPzwCB=w&
z0xADh=CD`UxzKlQK0O`#4!JX_iZFIv>^i=~1=;qVS~d70_rZQDyH5*o=ou=%dSUxn
zdvZMavkk(dwq0*Rno{g7a!pMEgkU~;Z+DLaT`QWBnrpT3dv{mD(#c{{f$AC#lwe@7
zhj%X-;%*dGEvbx~PK-c6{FCd=n+SPj_0+6=w<--t4cF2l4b#UjG_-EYc`ZeFurf(r
z*Zn<v3e24g5Zo3&s5bfYRT&>AyiSAkK3E!mhG?XR;boR1ZJ*c>kA|B1^E7KcP8M<(
z9(Zkim03O(TV7dNc=BW_Dcd4z{uebqL|tE37n#j>dh>i%PSlp?j~H(%2w4j6c~<Cw
zAq0&O&}Ld$S`xCT?=SClq~YCrVAMM&;UhuxBPmR8C<yfX1>!)QZJt+5cu<ElHwGw+
z(J`kDEG;Z;@azK}({P!Ki{YFwj{K%ea%Y^t$@aVy<bi@i9h$}A^L5_nJk_KLO&M$C
z&SKG-e)Ufh){}#^Q|C#956GWDp~cYF4LPFswN;HGcNig|>L$NiSqR%}r;b@=%}Up;
z?bKkM{(Ya9zS=b&FNGo#A|oR=k+JH04j!i)E3<u$lCUSekU!Sb&dWyeveTH4uZn=k
zn>VFsWsLVkDf+|jlG~xVR<z8jv+a@P`7Lz7{Dq>_Xmr8;kF{b<8#1avz!V#M3k+P^
zH7CzhHc6IO=6{J2m#<IMp2D!=XTck{Vv|`yqDvR(Eu&hlCHM6DNMF==E#UaO04hrC
zCxqN|Gq?SM*1jAcePM%$9`)G0|HIl_hDEvdd&7gGB8n&i3J4e=DJb0_h)6d>m(nHO
z1B!}(bV-+VcY{g^NJ+PJcf-KEf7X59&%5`t_wjyu*9VWKYt6`9b6w~8KY!V8;m^GM
zaHXtc#k5D2E?fc%C-=ItkpMA1y2JRZt<%>XT_L(vP)QlmFFNntal$gCJB~|UW)Zva
zAN?@SK}3B4(oZ<S-{0cZtHo&cY=f?RumQO4E!iIZ5xw4^pb)}Z%(BKa9Gr`}wtWpw
z;WBYNR}te?_Fe6*mpqfFlR-%^O+rf;6c^)uGLJW5VW6P2?PPv~to#lF_(7<=nzB{v
zZ>DGi5ih3>j#l1L;$x^ur%$M+iEIzoK^joYO0VsNeaZfD3876bOCJu32FM!fMLgA?
zr5(2T@sIbrEgHmB{{H?u+uKjCKl-xs@72MC>B2Bt4&2y4L8K~BrxC=-wB&lakB$j#
zRb~YLORjqJ@&A#l!m_ZqI{gRayw?SFaOE=COqJ|*LCdne@sZug_n;4ok=e9rLpv5U
z*i_mq#S4a4qobF*qLUQVb5-BTeYez%^{{{S5QdfB3^oVq8yecN-VwiDiM$J#F=7B)
z^A6jOl-jzE5}QHZhqE*H8Oya;^m{&#$3kSzMFhsvvu3OY6Gl<j<|v5C|2aQoUkLT6
z@|lF^Zm-9({Fx?iV^9X&B%Mr_;#*t&lbi3SB`5-s@>A2xxCn4KhmkYAl9Ia1<%IK5
zc<Qavxwee&j0Djr=rPgW?w9)$M1Q?$Ctr;D;uwhiva_-s*2cJ<i^>D?m=HDR8c4m5
z_2Y$JqE`elo*eQw^4}Zj>S9B3X2)<-s{O}Jv%?w*iu{nl-CFmD!8o$B9u~J(M^_%c
zpduh8A%UVmQvb&bnhXfI&NWin)cH%#i{|a!e;+RXT?8*KS5|DApq-I{g}>ml#r3JJ
z^)(Gd#y^t9!!?bA$MvBl##;pF1Fe5+B86S<%Xwu-=>!Gpsh;eUkCd9V=+k*1$q7ba
zwjlK0MAuLMBqk<)TN!Om6z;q23$P|IHQU>^Sf1yqPn~~MpZ#1C{62ogB}JrM#+zPO
z4z>vYdp{ZX&yk1l^tU_<tg>65US8h+d$v$TaGL<kr(Vo_m+?sNf!q(>t)~n5p#d^7
zAM!(kgKsAM?XTvv{D?X|Zg-}shdNY%(6+?hD-Y3^#jO?<zv7Jb^YbOYcW(l8miucO
zG&JdHws#<j)@;-S;J?AVGdjuL{rD=UrJ^)ysr`KcYjnj!*=gvK$0uPS+=z$o2gsh#
zN=-kzjEdUrooyE|iG{?XJ=Vq@HE%Cq5Sy}0S3gj#J{WDf*@)GqL^xJ#x-voQT7`t<
z_JLYXi52bsEPL4qT%3%*2!jms3wmz~Y>rN?zbc3;$d#OKpVa7}vG5~Z_iYcx_#K^(
zXT^2!10ly~zWEX&75%5NB@i?{P~9!_5mpLcm-LeG($ZCpUV+I)tCkm{6U;L^F>(dl
zqc3OPyC1J+!l><&NeA-^9~V2dvctO^A42=lW^48pRc=c(_VSdrwzc)u{13;`Ukv@d
z{YR6MD+o&)n%&jbe^gZ(&=UK>ITA}qIb4)KayM#*ul@@Lz~Zcke)hhF-*F5vFFDfQ
zES_XAXBYMZq*h?xZa6$kiO9;zdI?z_qvPa=FVa$=<D95-WjNW0P8e|K+|R$VLD|^s
zvb&NF|1_(RZFT(m`qdcWbof^<GEa+<@yXTqcLm&LL-84&nVFeouB)kNzx=rAR0APW
zrcXIJIJE1nL(Ol1c04CJ>!*^u_2RJNK-DZqW%HF%uzLm&6RVGmU_W<qa-t1n5D-B9
z>Q2IbUZ9#Y$Dm?YR(ug*WVFC8n5!tS{EUt59*0{+jx$>T12<jkpRBXk-16#^_aFT-
zE=2Nnwv%(J&)yRfQGCSrP<USkYtDZsg^`v=@Xgb^Ab~*qogZMBk$!pFwEmc~UT9)T
zt+z|m&$$T%fN|zLSf1eFf&RS7Ity*y>v%VkwfS$7626dlWyQ=q7m8^%7#g3@7FDKx
zAW%1D=-asv!}U@=1Br$jyy)mCdF1jDS^*S*i0(fFWo4H8E583ym+tNE;=JG7+*GP7
z9OKB(FwNn$Pfnw;w6yF>!0ZLQ6{AVt{EYK>C!eExcV{9kbTkt42#{qk5!*s6V9Ng5
zXua9fS%nVQzOT>dv*;ncATO_$R7d7G(5&?YQ+w<LuA19y?{&MquT#<y`#jnh?{o^y
zYu0z(CwtZH!y=$lb_g<nrzBspvg$1-Nus`H6{rR<<rM{i5I~u8N2?b~<u)ex!Npl@
z=&Dn>Z6xY^sr1c3UuGzL3!{#!VG^mpFiN?CEfBgkfd<zow~7pUqdt-y$ke<rKMRA(
zGcK+-bAb#QEr!Ak^BGxLxR-wFJi1Z(%7>bo`o>+Z&B#5nLj!bQm=U}hqmg?BfkvN_
z$HgsS$2H8$<%gy+%{``hj7P$=y6AITKWTCBGmhyPMhaj5^*vXf^^t0DJS_b18se~6
zuFqs=r#9F(x3aa}fqF^l7!HJ|keu^Y;5C?M8JL)4WF)BW?n%Y~(pw51R#LoI3NHJ*
zCd-odW~<z`^mX4Bhy=z15iMZ5ybC}v7%X0XxvMu=%HSnqR6ML?=!Hg~;4NatE36Sh
z-9F^ij*`#efHe?i)FmKTFZ~+{_eSJt!97I9J<FCG<nAkIE6r*YY?!qt_)nF;GAa}M
zC+^uecKxF+weMwxN<KP49R<2Q{blg2R2(dY#)L}l<#%)I)#z)#nLVwj5R;5_7j%?S
zwU7pZx-vS2i<g&oSbPd1*N$CW)<=rV!J3(oVZ4zy-Yx7z4Ta5oz?=;a-Ud%WR4F^g
z;Pf(-HbqC<9Bor0?t<E8%q@<`+2(kcq`^DTX>Zv8rA-m{BZ#)5H$E)P*wFH9=Mx?q
z>FnGbgR}iuW-pDqJKAAm98mc>y_tEC1E+ucM(d(koR}!2AtrV|!wu3iU<ccq^FTmO
z&F3T$@5r!C6y{X3fjz_ghKAN@{v}@-na+@^Dw@_-X}auUqpsik`@@xH<%Hi_#al^8
ztO(h;D-8~hY<9Lm7_OiY@6)8u^IT3g*iBvY)|7Xd<s=2=0WCH4-0#d#+l4Otz|T(c
z(b0#8&z?UD`ywwbl@(D|=Iy-@+ui;6#*)qX()eTs<cdQOW#vZnxEQWXSDseK6Oqbs
z6-!MJ04f?uT|uyyjWkPZyt)s=YRpYa<^xr*$KNKr$6|?gLO|8eaEcf{QByz6ug#Z6
zTSwBYd6n!mRr+p_9T-$zS(%TR0xRP}XN==&W9xQNQNlo8YkRwyOC8~wE%16MEXa}$
zd`vwHlE$!_O|{nU19=?>d;8w*KFyLhnT&7NV+&XWJQ|omj1;}Lw!%Pw(-qG>{&xIB
z+NYNdu3i8xMlxw@?)36pj-wzWgR>L)@@AoPZ`#q$os6s^(1*$-2&8_?0n&l+9s!Tu
z&lgE@6|M_1%cs$O@u0%Z&CZStJLGz{RhSIX1M+%P5EuY)PO1v(Ht<$4_&Q_MoOz)%
zGEOJQ>-hNjxq0UpmOXB8P>}9qO3OR1sZ#b*qt4j0i7&KBXVIg9Coy*zH5+wVBz>L9
z1_tvptoS-FdX8C(SdPB6wqC+aY4m1Z5x&z2A|a6z$Zr6!Y-nw?D?7B0oX06h;%VH=
z!9RVbTn}{M0|7sMIWDP9!RK%dClFcl_RLMxOMGTWg!DP#?1Bc=Nyf5?|5GQaC0<8p
z;^y+0GcP_Mz-Fh^xwm<ELcr8?6@VHDisY=3=nr!k(xeQ3;Qum)#on}Ws)5qtvE%0;
z-otkC6olvf&(qV%(i&wp6=R=8yff0v^R#NKAh!`ZFn1)Jz<P-tmuFmFY}jGxhEj#7
z`IAapL93zr#J+jjwSSUCu26G_kIv7j;63lYO-bYAdIHv6B0{1whgsyXjFPfaDO+jc
z4-r{u!3_%@QTIwz4Q{Ay#dSd^g!+xh>^j>6OZs}mU(j($*|F^r(*rt&GNCJB1Fj)(
zAX2N&F4I40D&GAhyHUCRlr%VIN6-P2V5rx-=E2Z}gY`Y5=QRM&4fqtshjX;K;@Pc(
zKxt!rGm=R-!;EPn^VO=Sz2`7nFUkh;b}?DHg#<m3y_?nD0>!Ors_K_7dH;%|x&PtJ
zTUY0dvg!LnAq)c>8#{;j2)dyNT9i`*0%Xdq#v56V0(*P<w^j!?yf-3G1)04uXS0+K
zKGc=;n+;4r0ssd`bAMCmmMi5=(>>UkcJPhwq0Z17AANOHJ>5x4xneOX7!okOduUgc
zxU9*3dl3>6O+ZBCW3a8mp+EaeKIJ7qh*D(^gbzM+V>A{zW24r<9oROTJUy+&TlV^0
zBF5@fh|>1&`}+pF{eO==KHL<>c>up;>^&;L`S-u$B&pypVE*}I_(g8Td*{FWZw*I+
z|9S<^gTCV_H2m!UdX?$v+q1y`yg@073nTWQH&CySNWeHQ8V|Rmcr@Hxkty@~be)pV
z{0%;h(dE#qh%yH)_`Ue)b+Tt!NJ@NDN!X^l&efQhG)GEVd%O-=8Ie(kwa&Qur{+U%
z)jmf>sX4u9EWAxq3qHDv)TE>-kV)6es4@708_Rhxo<E^XR`lrD&0@54w7{fyOEEnl
zfsb4Fuh1&J&d37WMFb9B%xitqV|#+lBWrh5T@9F+aju&hgdc{t7Zm1mQ<Nn>5(`{e
zUPjHaXcoNf@x)5teQGllYrDYgw$xaz|KrDlsav-eI(48vS%>W*>~<HgG}IdtHi&&W
zEN?E^BjaFqpULOmKjUmU@~Ex5DZI$~O#Q8n)Z^a1n%q=-n!;C9*VtG%*g2m0*Ept^
zl@0W?HvMTkp4;gIEj#FAGVrP#%0X|04RW^rTJM>Lp+y!{X1IO&`aXa1qYae0#F*yz
zD~0Lld6&IqBKld-Kuu_CPx_OkC*@7Nm1JezksT8Be+fuw$7xt{g*Kl@ok>HpY<p(1
zn|mENZ0ck@YTxuc456yGYb{Ci;Kxc>wT;F?e2t26FU5wnKVis>uyJo;t{pq>tX|m%
zqcUUe;_g1;bcjh}_CR4mBsHnE%if4PJ0E%EQe*OmO3i2xvD_!T#7#7cud|mHm)P7-
zB1m0h{`|ni%<u{0oaURg9@YEBxTYpPoKq|M__36+AT4cqxc3{>!vWyQ)Ysc@c6bXK
ztNGo%z(abzh=irL8lV*rznFh*=;|G{(plTvR&OtaPd$2RY+@Q79!WQn<4EZ!IEmo#
zqM*2ig@EuVr}G37H&&P3@9(_sUVmw$FrBpd*5}0q1cOR$)$u|C>_84$)k-Q#P*#88
zyA^QnT&Nr|S+NEV3(N5;=c7N=i0*+uH{w{uCWR;wG8BTh$jDM?nP9_$M8idFEm}hC
zE0+~Aqw96`v@O?LLuoPuIjzR`A=s@)ULHJ*b!TS~5=kIF9zR|U%7a-$-}|58#_%{h
zO8>%1f=J9tCPi`SugA(!1P(U-0p$3^de5|sR6~Kr+QA;n(?X-{TiUmb!u}d}w<k|>
zzfqBpIAhQo^@=^@{3y59WYGtDdKM#hoB3Tzj1HJYT`Ayfm(i25guMl{9fVDN7S*qe
zA&L2led$spGK}nQNu}e#y63x#`C7FuJI(eGmAE!k=stD=&6T@@OH)^FC@Cw`NqE-Q
zHuRbFDye1F=uhnDY8)B3iDtq^7c(_yV6}1ol8L#}ZsBV&K!wLDJ_X(#uG8Y>@k+{0
z&;F^TR)2N}WnxmiYzFXX6iE~uOL?kzz(5FGI<>fhC2P>=jnOi*JEWxhTw8^yA5-Jw
zC&0oNpPmMtlpQgnWftWQYdk(m+f>hdcxWtnDz6%=XfhI;m6Zqj=RSF!qE+#B3X*qx
zM=n3VDNqN7X$R`NU%a0^6AM7U6g9Hicxo`hhG)MAoYcjW13*n_ecLA{9?-JAP!`>L
zZoWDJmDm=jmNJVjg?KE_Pif7d-F|=Zf)h1grMp_*<AFPfU;dJ`#BI4RDrY|VWM(EL
z)JK##SV(tly>1B!As{OCuRYlpbUOUrG&~n5cm^HnjEuB!1xN3glG(J-R+2yOp{OI{
zlu;Hft*SC@{A>l+v*D+(ub=O1nLpg84vRz2rKLOm{A6KfwrWq41}&cNv5l-)i%;m|
z?59y~0v-^K3|9-GJ6c)nA(R+K^;%#5%~e%ca6)HiL9zv93`aYP@d7C*jA35}jCVat
zyPPCe`>P{QZYt4}F3c$?eL4nOANo-zzu%u*DvskVQw)tt+i+;gQl6imk0y;is}q1B
zzi5i{KJDi!$e<sK`kG;!<?->u$Ec|2TXrt!6fv<t3b4I{oUZtv1aAfeP$rGd&7Y2I
zm#)H|p09lpnh%L)BkEZ}7rowtuYQ)}qUcq*zIr#eQz>cbY%^1siNxn5*9#TapN{5W
z<%UYrP*Ms)10ms6$BmIa4qgrUQ8~iMvCEHwC0YW=ChI*sLZ}?q#*Vh<u8Cd^ieau?
zw5#&a4T9XntNB#CypqHdhllJ=zk(*#-BE`qlF}cHR#rqr;qI>cZu47`lizh4K1ttW
z@?PEpjbw%UN<=b?Ip02gW`0_NSZjYy1*xK)X2fWHwDu?Id}cLCE~mpyS+fBu8k*Dk
zI78T;z0_?^J?v)3X?y}f&OvrLViRr&3IxPku<kej7l(&2GqDKW;)&z2fMy}?jY%wo
zSL>>=UQ}evj!CQWXjv{qm81%uLDrG2jkSD{ft1p2Pl^};!E`O;3)a?TwATA&uOvSA
zieyqFB>bD7o5E`DbSi><$=R8;wkBS{oyDyAOZVSnQ5`CA(nYcy4Y0bkg)64X!08#w
zbEDZG4D2{Ju8$R3xC+VLe16k+n!v9S*7s}_x==6c;iHWM8^?4MC#)Y&pMLfryn?t!
z{6yxnEY4NjG0-*rZ1N+yel%$LD&$LzN4-uxR$s!o!;@8;(|sGCa)6MEs*#f4<%5KU
zsRS-UDwfOVa!EnZ-;GG7wMh@5LCf)sMW?2kzd;2syBSKw(DTZ0rln@Go9(ZL3!-ll
z-W7Ue44Pl%5^B0&Rdw~X@#5+)UC?CCS74RznjnDqySO;Jz77oNG`C(7qw>JljXK3J
z1jq=SuMw|8v>w;fr*zz{%=d-FQRz}~FVk5spxqWeeR6PoyEr;1Vyzhx-zg&&cvhx=
zv<<E6H%R!tRqlyO`~n+^)kuXNtbyb_b|ow3=@}%yEW8y$*jPDbWhFI0`wV%GG&0dr
z7BJYNecOCt{&#kDW_R4qr^V{<H3LZ}fG0^mp2Nuvy<fe-W4D-*VHz8&1&4iHysPno
zoH>E%-l?fXh{fgQPaQGUAZpQ}+yIqDhfaa#%M?0a25#=w(e{--@7uie$%~7|oCgzq
z3AHYJsIBcw`5)3Vm;rn{R)D*lC5+b+R@lm&c~Q}qpDtg4SXA52F=x%Zs^qfg-AhaP
z+T(TZ?G3#<F#K^>b3sNpb05cSfV>Uw`-hjDkKip?*x0rYcH;O<F639JT8{=*R8++a
z*tfN~I>=jZInFfGn?40BCMn6b%xau~lmNW`ckU5LG!fmrYlmJ8F6a&_VyPq5kQUF<
z)Eqx{abnUe_&(WHUX*Op{5wjb*}0N1JgL^}w5n9_BlhKy5&N>WvC7J7t<#x6xW_29
zF2`}1Y(paIPhpV=W}SK6Nv~xbh>gvs&BwvJdIRr<f<Yk&v<Z#MNX}P>JH&*$c6Q&+
zR(s1gYF!;;Mz7DRm%Lf>8d8PJr-Ra7*q}7-PV-NGojRshriMmeWY*-OB)q%1`NInU
zS-N2bOHCWH>j>D=BO)WO9G+-Z+5ivID&fGusbf<KCuB=Q3If8zUfFEOAP~OP#*b=h
zZdO%Ql?(K}i9I!tjhwA@P4GZNVS##`o0_P>rT&%7*7|=f2uuxzv9KWQR?I5=mj%K1
z1wGvpy3EX6tPeMjnU5qkFt#h>j^F?48K6Tx6}xc6<fdmIoQ+q1(oi2Ax=t?O+b-|1
zH+_`eG`BpzzuYg0A`xU_>g?|?t)$#oTD(i-tRqKvUwnIhk4&a>4|%ikp|y$05)>tf
zhnQCwKjGjYM!@P)2+;zmc%WhVNQo=>kA~&?Wf4WmY_+SeWF)2wxIrI`jW`f+rxb9K
z(^9m6E0&qL<i9eQF~05VXJKJZv^Lc_&RaX?${HnwSnMsgnjLq2qTs{t%A;NDB1#{Q
z`{Di9uXnF_Q|wVcBBr6{N1X!>v}&D56>W~!$4c5)1F2e(%sL<yqxZ=!$tfvHR*<9n
z)zM1F%$y!?xwy3Gw6V?~EcYU0)}L$;AY4M~MAw6LwWz2*+|OjleEah9bO{kAfWjb`
zFPMss*e{Y<gOr>+l`0R(tQ`>%c0tI&AtJACf~F7YHkMLV#S=m`X=tSAYOKPWlb0vs
z21#W_#YMZjd-cZU7A~6naj}+T6*3`j3`7mgrAk1bmz?FiyZHm+UqBu7iq<72e7rdW
z4`H`qnGhfGa$FxGXf;c8pWEI+iHW|1O6cw-JH+DJgtzIAZ(Bj3VSnUU1)&L_<>K-p
ze8llu*DDB6mQB{W8ohoE101Q){%ASqUMjh{sl<In^>4)Nu#bL94!AE^tX~*WHjuRG
z)%Tv6zTS7_<%l<_$6d0~C6TLamQTm499+ZqL#pmtLmV8$inEtRmqnBTA1*9>0WntZ
z7|7#;Y;0|%qi0dLpFt(g!iUy9e0Q<KluV+rxe*sT93Uf-{7FcgF8O6Cs&6eTCZ?!Z
z+~(T`*>(Lo_wore)YV3YhAw;uQXXuQF}#*+<8i#@{3m-*10K3GJxySbLD|MUe+?d2
z$U{(y>FQ|R+SyOb!=vL-;h->#F_=O-0}6-r_h<*!(3fqTmiJ(ny@>c0R^z_%7{Mni
z-DB1JvQ;1kbwk(8EGFot4iZU2>;sy=457*v=I5z@TSI5HYh5o20YaSJsGk)B`|HZJ
za6Q|ul;}JkwIEjjUjDt0F6+RFvrb^5Pk|Ahn+;vJIAe;&7g<?FJdSGvsTv1HvLTs5
zuWuLs{OJ~g9$juMfk2)b)OeoO70!b?`2)+)H^qTB0pT6Y*SY9fQQ-~dtobb4OX2U+
zJYw6Y_U{h#7VCH*5^u2$8>QawX!~tmQ~QeQ8!9-6;5~@9$}C2uqgm7x75%8Fv7g_R
z=#nN2eMJQVBTM*>4GpWRt2c(~;v|UhjH~xoQ_JL)<@>u84pEJOZpS)u0|o}lY!#J(
zm;o!}V2Qpz1IZJ82n49@U}D9Sc)UU>D08zBGVok>?ZQI7K`(AGjck~?Ce*gCsXYu1
zq7sjr1gsF2W`zhu2fo#3^H>B^<Y-Uw%*5&5f`xLwGE7=E1V?`N`zn{;;~x?Dt!y;W
z`AU|(dm;Si>M>1U<f=o$hcLPG$04j%Q$#pq*N`K3-qb}?^9Bv0dB6uMEYDWmUou~2
zmHm%@t7*e54?q$t2Zzs>8hnJNy?s(*9VI+==<f7rCI*JkF6NbiT2~pF0EnAR@x~7i
zOB(xQ>Ehn%FB#lC^C_(?O^7zARwX65NK;W!Z4^3#F44WBvR9hAiM{oo`(DB1IqHWP
zJ`SxZ84)Fg%N-nS%^OI0ubw~0W<!~|nYWNW?CfkL*VA56CASrnm6fHb53jN3@gQk$
z{@PEh@nH8eFSifj$%5k7@_KRsKFgnnlqj^k0@Crgk1osW#YA<>+*`ww!^NeQ@k-|v
z5U2TSw{IU^{p{ltFYp>V<O51e?NwBW?Q@gVI)t=|GW<RWt3?}-YxWctZ9N@j?CfM}
z_=KO%dKZ9y%n&|l3w1fPZWbsAN#9waBA7ePURpqC<{ek#mkWIme!xyh%Yzb~2)xHS
zKPq{LdEa5f7Bg{5%#Cu-WHFz2R$>j1kZylJHdB;**1yYQuHudbM^dEw)=_<-Po^r>
z^p0e9b~X#EHZM=*g}yG#=v`7n!#BUm;64EnO=aZ<wY@!w)gRvBk)pn#hk84P7htvq
zoIeyM7&a4J1fn}OHg;p8vUit%^C;Crv*IlP-u@U-euAUG`QLG_Tho+|8)FY}rW-SR
z6XcBLgM)4FFn)I3{Y)jkjMK>cBsVwv>h6KrH1!U2Ah_=C^bPa`1p1{gxq8&GprUvp
zHK1D*mNmSDYBPz}imIy0sy)apD^#VT;I*tl?QF<Kgoj%T*38R^h@^ydA;S<TIt;7`
zD+75JBbG;`q|uiyU4j~3*niM0bs-gXFWlL)$<MBANlH4d4xP>Kt$3`m!)LjyP~|iw
z975%xS?Y@mRxm}391uQVqoe0%XW3SoLJ>EKoF{XU`S47M;ZrJD)GZkdjUAibizJ{7
z7vipmr<efyxq+OBR&hzmtl%g|bpI_x;+6zb)Ivl=%j@`K`<x0cwy$qC*l~(oPI)xa
ze0`%y{dC%lY`H;lphS|DnJJGCDX!unY{{si<g~Q?B>V7evyAo&?Ck8<aB=J8NKj{%
z;LiuZ@s>CK=+F@3$(eUppHV6`98Pz;+9NSFZfbBk=$UR(8&-SP(?ccR_SmrKHwtwD
zc4ap=&B;27fw-JUf>JDV92YCE22J`Ny4{F;WVQnfENHIv3lDb!!~IQ6%&vHUV)`n`
z%t_PI+{{5sr7AueBx&hoqCy%S&zvL&D$o_hGrcI!<Qe4j%m>f!Lbnwajc69&nxitJ
zyZBM;wbwPiJ!0G9Ti#(8U}9L-k!Ef%?7ssyoss+29>)B}V4j&{nlJA2ZYk*wdWJKX
zp2ANN4h@1P2(8t%es(s*_g_O{RH@4P*0w3O_O_DJ;!zw1+q));X|gff3#pP!5|mgs
z{X`!>evAzhrQPy_KOYXU@nvO~QGL*;BRG}4QBrDeGqHdWg4lk9=iZc3inXTixAgS>
z{(%4r)_FLC_kH-F)9e9N2jO?^@%FfF7mB#esa>6Jl9`zpz|l)?c}dzgFK@ka=MD^!
z0=e|CLf+-G0}c9&P5=1#_{=mV`^pShfvEZ7RjKA}`pM&BRZVjYY3@Jx(OXPuKOy+$
zSN9=4Cjpu3@v2YiM`b|o5Q!<GVw5_9)Yblhfs`zEw9jwCS^vnFC~j>vnh5FFhkV^(
z;jKJPzdX05X`jbfV5Jk-Yt*&0xGdyndJie=hAz-+$SmW3ofa7$UOH`<7#b5P8S(}=
zA2=#^>1SdI*(ijhD!aSyIXLPRZ01|VogXceAEi_|EG^H@x^1kF&qwIlu4G$lN+T82
z736|8rk1WEVDthgE_q-JHCNWtxK*G={`|S=Tsa<O7A-$`dj~lf$*fJg-Y%P)o13{Y
zK?Y3#rV^(1()J_?UCdAH?QAZ9F6K%*aq!kr>dTkSh(ypNS&V1ad9^g{?QH@+i2eNc
z*Clq--9={Prv#5f)v8t3<>8FgRr6p~s_<7>x?_isOH_RkkzY~q0Vyo;3h_5`+y9Fc
z!c!)_J$?uo;yLcR?{ad;L5=4Kbj;JYiT=534ftjna0-W8*so<~Zhl_j$_tS4?~!p@
zbKRqEY3oW#%~VDr&$Cr-zR;1ySMg$a<Ujfe{pfwo<z7bVuEFdQWPnVk)MG_O5k-|x
zz5zPGX5&)|d<8V$(D0?Tyu2iNeWQNxr>d%mG7kTjDe3{qENGk8eUaUV6*(CmaDT@r
z>mU-7GH+4b(mR-v1})kyWlzWf#&vH?4rUBv>>r8vBmxSI1Y#4_aa)hI{F}B{IfD{!
z8@Sb(ZwY$+&}F%+tnAUTd4&EG*`*j4>zbL7_4&ik4;9`WwOBX^OUqga?|5_qu@__-
z<r{DnW6CTIvYx7CF_J!CnOl#Kiz;>KFh#tQks@WW8|t}D(<t1Nf9uw>XFcNLK8Ex2
zKqv^MJlEE)3%N5wXS3Yb`oD8}GZp_Y)0yXSvR=cj2U89#bR^s$k<)qf@J;o2oqe{B
z5D)_?;#Zwvst%UJMz|hO2F2vVf$~ty+QGpAPc~<5wTh6Ku(5^zxa64Me)(Hq;3SlA
zL*@%)?e}V@!n0D?^K9My_^bxyel>r#&*!$@&ioAumRZPek)9NBbBk5e*wGoVO6)Pt
z%%p=1a)lD=J!2i?A&5p+P7*paYkWXm&F?(@p*(*BfT)q>Xo%JVi83bxlf>;6Nv6t<
z#LBhJY05k3i|C6NF79mG0F5Pe$)T*+FW}3!_y{Fo%L~y&gTtULbUj%eE7idL1M$RP
zzHs^qxE@V6egCWLO#umMDEIlkC7;_g28M<;&ZjOb5gpfvne?AVQ8(4#tqd~wnC%Q$
zfNwdAKFjX?L_?twN?Tx0#$5X<Gc$A7_wPdc+Uthldc7->mYNdB+fd)OH1I?NjYf<4
zlWHkz)jH3-v}R?tD|<B>l)ZJge=MN;YjRm|bO?)US$SopGX%V*N#iES)BjHjQHcZ)
z&X!3*-2w9r2ILu--4ZEoRXOY}zZdmW$y4uOn1Yldg~@vnK{u_;^^5FJ9IlF|rPhwr
zTWv%#G^A||K<MT2mnB4Ev&v|N$+fGqXSJ@kr)MC|=fBa=qSG)@KL7DxraAD6CuHXE
zyX*$}nn8wiea_@}=9%;~K30xr@*xNymE7IE1Y=a(nd1eoc(zz?^|{_7^!K*8o%m~B
zI_ABIFpbn#uzka>FJ2p>vR+uUByF}>pa%f`6$EzNP@yXjuz-<DLj#qRQ;lBlJc|Hk
ztg={Zwd!{is}&Rq*^GXIbFq|P?8RTaj4uI2bDw<YB`W$6UtizO&OV&7K6za%PrKS-
zu#^0W6Po@BU9IE6MBccoshMdC{HJ>Y=zWW_F%~7Zebm`q>hvFM+<AuNk#qa`X_2r3
z`pKl2nV3E!krKj5!VpA%l;&josA7Wt;Dh+XnvPL-gGi*<;H@+@Max}53t&^oP)Bp)
zlRAYy-Sv4>$d;u1L|B@E9E{<#FrVIs=ooI>dA7Rp>%aq<y5lguR}mLOwiN5s2Wq?N
z8d^Hkf2_NPI6YWeUY?(un*plp&1mILlb;8h^-SJ)Ri>G-TsGpIHR@~odx}vphGM0Z
zm;M4M^w6m**QglI=>q&QCg<qd9XJJ8uFQ9UrcIUfA6Clizp+w#|3C6og0YxK|KY`Q
zDN~vbhLTcJGozgQPI-&Y`72<S(r;l0B6r!iGof~OAkWWv)S=JgDyh7-JO!D(PFZbU
zUVKb-i`U`_HFZsdbj)Cm8e_!BIVAl6rj5`vso^T;o%FT0aCd?Cc;2Q$gTul%Fz_T$
zX&LP1Fe5?H!ErrVWK`5}U^cQ?NexziUoJOQ;`1^^JiI*u?N+q2E#{<_fz7;gqduX%
zqc!+s(xf*HIoKD=?T(9Rmi@XoTwhNrxwjry4bJn&;SxlbQ0Hf%UZYNc7tMbho1QaQ
zbAmaWds}C?;oIyprkB*qyqMLCc6%DGz?kaGaY#U}DNjs@_fMN*jq#5=^s)dl0L6Z0
z-Gq;Kk)4fwz-p{UxAR`otBl!VX>JsIcHKi1Z;{<rGGy=Ok7b~=>(3U0AHj2?1AU4l
z37DO#1cdINSvmr7Ef*I+eG}^g1I=d)CYe0|?CCs-6_t$mb)gH2`#@k0YGEd=eSYx}
zHvtUb`Cy_jGc!f#?nLCm<qx&s+aG<_$Bg~_9uIoO?@kw|rRrp3))89*H-!S1d8buN
z03;bkLiYNFa7sl|8wcy7{n5hfmm%Dk0h}%>1cx7ww+9(TDvwy6vE$+3=szU^yyILK
zTk?iYfDtt>J_DMicq*F7I`@@ONke!JAYJ2kXXo)OKO6rwxQ>9>_V*36IY-*+VkZ9a
z{bpuH<<}=TkCqV`%(Vnk`nm3A&>fKq=zpkm9{toXD7Lyrp4Ra-DK|APc*_uMa0YGy
z{nr6T2X{Pp<Yn#&o>6hDiEG03C4u=U^vCDGz-xj|aQ+mv-dh~DZT=D!t*#DXr<3K5
zvND~nOi1^I_4+@}Qw4*0>ZgozQOOm}P)%sueV->TIONzIQ&;;i$#}WM<n+q*Z)fd^
ztVNTCn9WlbQG+3v0wSfV9oFtJst*;6Gig_`2{L1TS=B3ez!-jXbmF*TsH3Q1yRjYz
z!cT=Me_7sYyR|B`H_^l3&m({PT5oga=^zOsb1XJ}ovR&I4?n16E!LDfS{=|Srkk3Y
zh%Zw~MlyD`Ix`>4ND&bUdSC#-9aI@aZ?#ZzTJ_QFRTsX#f=}`DO%|V|bCp#WMWHHV
zIYH(zw4Xfm`a%2T$#}h$Rm04q$cFJcv%&ssBnko{)YJ}JSkTCl*;9|lp~H14A5-nq
zKfQa|oYwp}E%*R(i%{XupEBm=VEq*}ZQEmr%uI-lb#$)uApec?-f%QkQrf;h^4D5I
zBy)lw=43MP*?dBA1FXV;6W2N&P3z?UnumT9%-n1(M2iE|`}5-t<wx(!P|<p{fC>K{
z9Th`&90?~^jg1M2HO5BVL6!sf_C)<DGaJ>=#E&1fvXr9lbo%@jx;QO{hKly~Favi?
zZ7^GwCr1LgULN`fVVsAc=d;_jtJBQ>%4V<)G?M@><fe}Kn+=A^o=si52gQn=ot+on
z{cYM==u3cNMbM_b!JHdCB$Z2Fw*>ShWMwf{Gigx>INhYA<b?abC6FAXza%GXUs=;-
z70GHkI^(%eV^yVQj8AgkU+~KOezY?maOxmsVPT<a{MmdPAK?(U#-&+STkDlYpY5Z(
zW1{|sj}Yk-`9w`YTl=h>6zL{)^lP?NQ6&$pH~c5{HI0-U#o=UM4og@}zE%Cn`tDLF
zRA#cVvT{CS|JRdp#_Uj)cHQ0IWMvWi<A>F;!f~_X-Pzd}82r#yK>Yy<`yI<+F*#A2
zq^Ea}V__%sTeHeFycpX<xx-a8R)7gl4i6Qk^lTT-wGV7m=SU=EM;aBX`tIwRjXH1s
z{UOMxK}-l035FbRe#wbym>QdY_}DRC4Qi&P-jD>Uqbt}KKxVeY{+eg6*!Tzf=g(y1
zQa<6@kEZ^9IIn)(SvWa5Do|2;-RyP^(GoxrzV?8JXIsxaCxpekDs4gX78JW;JcTqy
zJ3Urd6w@S9|IYB6wN8stoE`)4dF4`i+`St#+;rj*>~psVVH;|0Zgi4PK&{OC!*4ha
zzKFIuZyfsEhR7Tan9xiVX&9o>7r%dfHkoDH?{xu<%va~O{OPnTT;Sy;f;qcL%&a}T
zu+X7k;7_rWn+#}FAMW)V>h))lfZTuhfSqOWODc*Zyy|WD7$aNu_4??D8qCJ@yf$6N
zE9VGvJ-sK_axybr_SrJ4s&EmIsR0#SXA}7Bd%JiF@dCjg?G}y06JpG^Ngm}`EOdfu
z5b<`QGwz4im3*~YdO^X-|3Z-FtX_DtA+V0B!?L@~&-d^G0?LH8&T4<E0Pp)hs8y0*
z{}(HL@%!ejN2BFtVn$Ex_Ux3u(t~N{C-aj_o>tIT!}a#2*AIZU`3NbPf?uhBWBMvJ
z55!b%ZjP;uO_}9{rnWpFZap)Q1Eo$#Ks=D<uQ!$==IXF<eww-*NXj2Ab8};&78qhb
zRjSLLlK~XG)@W6jIX+p8j8)n1Y^O>h64TN+8rp&JeUAHQC}R;G4+Z4>-RsThkJoZ@
za-8`l()Lz>%&rtS9nBj@U%CeB*xkL&oXkAXCnH_dNztbiN}wWvx6+tJ^?~Ce4|sZS
z?{uv}rf{P02hZOjbl<$0UD?P_PiGG9TN|#8k?;)2>qH-2D=jGjX=0R=q2^c$xHvg#
z{5gy_Z$fFwvSa1m-1IV!-Za(wm)Tia6V*nX_N}%v^Bn-`X;eGJy`Xmv`C4K&*!ih7
z=BeH^tVN0<;*br!K3R`clfQ~U=A)OE=ZT3(=r{{RN1CAoM$2{Y0%+sM;zrA_i3<N0
zDZ($NyX6*xOl)kc*3+*c3WuM+7Gxm&veL?Z^@L6RyYo*jeSK=3ELMM3+TRb(E~ir-
za>V~mSC^EQh%Ef>;q1QR8QLNX+%N5lPRdI>$nlm*uWxlKHX5{imq>u1Qdk-S=h4B2
zY{+b09IsAU;~uZ*DSQEXu*GCCDr-Np0tued##0uS?X5j<s6ihXxBKxMDJ3@aX?*j)
zxa?|Bh9r-3<h>BJZf(*p6#n`1w^RoX5C2(DN{Uz@DCk8k>b7)Qj3)Sk<&0rYgc?5{
z$1+F*P_1&Zwzm)epFDl$8N1`e1#*kE-#Jo5k3%IqL2B(^>%Rks7~-3RFXCL{`5iye
zi+pUUeEB+T?1#?(;g50o?57*oG|Dr!HV+G5H+bV4WoE`<PVd=oMreFevO}Gnt&NoH
zfd56Tb#ZQ@x}pLapmX&S>)yRfNl{c)eFcYPQlqk|=sS>GMd_hQr-7MXoMqGfF2oVz
zKL0yPgVTzZuAy1W=VgjJgmzON2tc~fsl5wO1aPSH9r=_KBNfeJ0Bz$%<BuQnpQ|}q
zZ?|L$!nkuU;qUBhY;Z6ZVtlO{Y{_&adKrN=?k6zsLuTvR@!B8`b;m?$<fzxzD{<8$
z!@OL4E{8(m{lCaLXX7vaT5FdUOYemmN?yk`bxqa7)nQh}+r4lL%C#`t+S*FvNqgu}
z4nxs+OITn=MwSXv*%AEVT^-64w2@<Ot?+WIr#czei3}q$6{z!AjC)^d#cBLhEc>dY
zA!amzK7tN9uzDjlp{pG(-z)`jjQyVsTAlwPMVR?C2D~3j7$i=AgTOc#yg*Vl8&GQ#
zNGIKOA+(i`5VZ+)gNRM82YQXatTKqnoHr(bI>Y+g<hKBjM3qa`Pe)<IQ#KYfC~39a
zYg@LquHfIhFCHS3esbmV@bGZ>rUdB(u4U|So>n#ZC)3itA--=cbdmF*85BiRVK5)d
zS?e{DMcij#P*kAl>F-S;u4fBqZ|c|s-F6KD1p@3%){ue$@uI<GcS3*g*~}uBD<}^(
zdp7kR3A7zTlduQqX{7-pGn%PrYj4A+7}8PEzKoDf;41_LLyD~yZ#9$m&jzCKq-0xp
z2p=0AZD}5eNbG@-3n9E;(p}KBNcl{^ku&AP`!;}0<!>!5E#+#~KBi~chR{l>W*zta
zy_H?~`}_O*0g+|Zu7AeD;t~?ecu*4BFpu-$eRBtZcfj{rTpt;$v4%Q;i5lzIt3%ch
zdD_+17UtUdxZJs28d2J*E~52%-#j6z*bqy|GvCU>B6sX;ZgtuQhEs@!fN>m@A4ue2
z`S9?Cq|LwYF7|QjDJUezY285tY50KhJ2e%Z{8JbtxVuY^uUp!-&Gby5K_&E%YbxJb
zM7u6p`4c7&_99Ah=L7NS8SwmndIz!z_U)p7>n1#SmsQnA-_*!N#&Kk37UKDTVP)0!
zkYPceUW7`8+oS|iDLrs_&5VOiRG7Iy2%S6?@l3u4^e5u4J|94i&HlCmO90#=|2F|I
z)M}6ms6>w^F);!9^iS&P3I5_pE*T=|+qd-MUg*ZmRjorq{-?-gd@LBt2A_LfX4WdA
zrTwl0_v_H+)Dz`9`#1JM#JL~OpZ!G#WKbI!p;X$>n7ODJr@NC#deanWg4Ltdp}Rr*
zr~&`^J1n;M98Q~Z!zAJF>VCj5rIN47ZLrKnq)dbWjEpp{69@X2$w)`We<WGy;G52X
zdMtt0bzpKZ{Yw`mms3zOmAH5jsNdZL1iyD31PU@89v^x1L}&#?a`v{bEcMb98~(-t
z|H1_rzhiGE+$12ZFdAI{=y!+p?sdf1r+R-tc;I#6VrLwGFE|=nAG(U}%`Mz1aOrP@
z8DSFT_jYkg=n_+`V105C2qg^8dHc8wP=0}Hs@nQAn(@QS><u5`cir9Huf+cvnF5^@
z>en9I9n}3d(MUEyebnv(qVUz~+Fa>!3#DKdGpp^+Zb=EWK%WP#Lf5^-F|ycKk_og+
zMMXsj9q2@0KDu=1k<o~TftQ4o^yuVBCZ6pbQu8_jAPuCP+|7G;D<Gf7Gx7fW`-BI~
z<#iPC*X}SvnLut%Ml9sop2|xo7K1qc`{V``4~9OLwG4hAK3ZR5Z#Vx$wZNz=@9+2T
z-)5=mJrSleZ~zqe?7s&`(zyluv$J=XEk@YP%550r<uz@fK?YKYAF@$#v2nc$Z!&ed
zSVrgN6Bahm-@i6o>I%n7m>B37n2XDbV3da`($Dt`_(Z^Ywh&vnQ=w^(xGSiq51orp
z{TQW~zFK8xV)=u<Z4iu1klRN<Oj_i2#B$|?qHTkfjUr|C##xi7D17+rjGV*^nXQ7#
z*Hi{nY!^P{9$PnkA<-s{CT(k%yMGf3Xg$I%^TY}0{e-gg01rX|XeI|MbbBYeAm*^Y
z5vvj<WISjI((`ZzkKOI))Xb#J3g-4D3D5(tudhQjOZ;u(Hkb#GWb{UZ)>?oG1ym7&
z>KUo%S&9O^uf)kje&hokTF-FQ@i=6+{+Og$#=U}uNBaHGvb9mV5zqTLG)9pXhhkT)
zt@AANB(h#w8WP1_WUFOv2i3wr-RqWuphUPIp@Z>N5{m2JT93XtClczGux9v@Rr0jQ
zApl|d{reE1%GvqsCHEt<h@@mUpgjMKwm5c<RgRWhWM?UqeX~O!UWIIlcUWh@<k~w}
zC(%D&*DGA2@t4>(*wC%FT$6q9?p>KfT&9{gEdIcdxN|}T&)@IQ$xv!QKc^e7hG#FS
z(b6-1Tjwd@UJZU!?|gWl7(qqQ3W$)L?FmHwLGQ=Bn-gi%L{iI0f3}F%d!AAIrJiCM
zM8$v8gSZ(#wP7`T^M<kdHw}UBhlfcJj0Q<DXMClOR*r}QP_<-9Xm0AoG+UF9khGq~
zBqUgwn<vxefwXUUXri}gE9F`d<mnR=N5~&+aB^Kh-5Z4OI)g|G?l`C29k;{3R<HXU
zC;e;fmL9S{h}8I$_*c{Je-RQ5<+v>^ElbC;9i4#Mzw`WWk#hggGIQz5WdLF%6p({Y
zg8@tt{rD05gw4$ZKmV}6VWP)`CZ>Udl@(x--a4-<7cY2$O_{Li5<)0jCzUo_0{lf}
zA|j#-=DyL6GuUz~IR+vL<&j8O)sxQ*-n>x?LT<`ir>3V;)6lpp&DbzJW8hWP5~&sO
z4mfu$_Rm2dT~R~J@2KJ%<hU3O(e`#^Z~LasK{!Ot3{-D;{%B}Bq{fRpE*>`L*Q!0h
zqxe*~GP35}rWk_Wg=a<6UgS~AmBK>ATj;+0WJHtzbM6VO5wLkEm8+HLz6&vt?TQaL
z-!>&uFC^W6{J8l?lGtg_VpIE{5D@KlcK+!fOd-`*nstAk>PgiaGib%dU0tH|&Zwp}
zQGty7c7JPg1TLXplel;wR6vW{%OGeOm}8!~(}R6JWHvjJ85$@ITtEB!gYosMEbv4o
zCKo+ZLVH0I^C|>h*B<%AKOi8h3pvsbKNHY@&Zb%0kIG*>OkyGv{On^fm^XfB?V^5g
zMM=r-Kt~LfxIz@Wac^^9Us$v~_~W*G&s8;8-<zA~ds41I(CXUgCz>=HQ<dlL%F0M*
zCnsYwlX|DKSoQU!F*ppvLacG1$AvoH6^CHNUAw)C<uN#?Jd3(6`Ys{As!Cl$qkXvP
zW0O88ZK>&=$saZQ1X5lUa!`qguS40MAd91+qBrG6t9-CLY}@G(N8q!0Vv#E{btUt}
zTZx}SfDlECzOU@_k0?k|9s@yJpm-s~p%)LijxTr@^%qe}O-BtK%U2Oyii*229(kDs
zt?bJ>5aS22KEItpu@Ibj+AEe;U2j<6!a-{V9G)&BK<|Q@nCP?jPuozjq;B(<>=6)E
z)|X_EB!d!4E+<`&!avx*$fdw|%D})NAt?bYsi+zWQcDXd2Ej<5+mI?M;5yncH#dhq
zlji*?FK@)=*{SPtnk8$C_R$f6PZK{mIqVYj4D^oXlWFL+yV8^?gq5}{#Nr{&&M?c>
zmcr7a{Ru#d6IjeC*Yvvo{xD(IL6y8Mw5WCf@r9!RpXSKJmV85l{fY5vD)IUG*_tkA
zD?NiuW7bNo%=DC*+UkQB!nedjI4w0D-@cXW<H&fM=Gz7XN<7b2dG_5o5GnuZjT4Z9
zMn<^hU8-CxuH#*Ozu1$q0LqA%uUlF4g&}i4;&>fZUS|SH04W!Hi#?&4$;la}vxX|`
z+(Nb|Hn0DwWVB}QX9xCYD^PuvNElL-IXF3ydRgY0_3`b>2hr@3@`;+h+x@$BX!dqF
z%Q}d2`gDUVEhW7emOQYow?|+SYl!tzQn2o?DUZ<b#gFH(V0#XIRehIO1kFxB&Ve&b
zv_(R+g&Fabg##jvG%)R5eZC0M*H(V8D@|L81(1-CC2(yGmP!J|16jg5<BqQn{%K|8
zU+)olCwHINkCKK;?qW)Au7$9$o0e9bDpi1=pQ4R^VpbL?wtK6~Pk^xfN>9(O+|L+6
zkGO}64^9wx=fEdAbr1Q_iRM@JW-36XY-VZ(#FvM5eO*<S?{UR;Xb>3H-F7-M9*LpM
z?s84X1P|tGTO59IpQ)f_K;}*qJ5H|b4&sw(rt}RCmM@u;9Ub&XQ=X5!J@GO!@YT?W
z7ZoM5a6BFv9<GF{t2eLD1M6WW0Bba8*eT5-?tfM^>O9+=34~<7M3wP2{xwD>CKOOl
z(aPE2zVGSjkK(bYO0H--1ilCzJo)7zPBJoo0*(~)^ipzf&xc&j(;B2D2KIH(gs_*g
z^E#|Y^H``$+wX~qeb{IY=pPuUgqoye2|~Zddh&ongN9?Bz<cCR-v;GCViO17g9mjL
z85vU(=ewK1VgBp?F9B1|Mwi8Pf2hJcME~jX`P?w4<ddE8EV-&m0$Lio8kLcEaF;1N
zVNp5qiUtxqJbbUc1b!;GCo(1ty8K^JQN<-*L_{{MJ<CZqsy{=3X!gB?!HS5Ivf7Ej
zJ;Xq+mgJM5%br9KS7S9uf#}4b0@O3`&)awUm_eteDRbJZrVwOkVCde5f$_LEB|RmK
zioLWnf#3PzN#vpk1@*TNi9HCRxNN4wp!`o=-+xJ@$vjNjPW>!L_3dT<a`ZCDwQqRu
z1a|#lYl89#@9X535zO(J|MWC+L@n4<uqCdIC}Jq?D-$W>J&{KIwdv0sL?IBR#$tM`
za&kI?xoWHq=!!L7$GO=$LAS;3n87yU0+3$bjrsb*VxsTw;&uicDa}x4YI15@i&G+E
zG)Ax*PIBPl;ySDkJp(7w-=H9xR@wcHNey}JB;)dF{Bbei5IfRbr~;C=Zp~$WCv<<*
z0-`t~8*;O<LL^cz9bKXvuQC}e8~;WZ%FbT;N<kPTOH<AHv1h5BM7Yb3QnjzEOx|F)
z)Mr-+9~aZRV`6g>IsN#c{H$rthF9Hgth&C=6p(Z_cDBmp<%daf4lY}PvhJJ{xnmMx
zWOwpE(3D&JFWMvtZ8|E)r;u+0h8d-B@l>IkAO0nxT1!9IP!-pCTouf5yUpQoH8$Dp
z)ZuWm;_8gY;g*=V*7%ZQdRW3boBJ09C_oY(kgL3BY1<yur0>q^@*BTB0!1ibuv`LN
zq^$x&|15JVDwXfsB<OW;13Yc2LLwqS!OCnP#xuPAuGzv1my*YPD3qqvpK_zhex~_r
z;bdyPM5sG@YX%$4^k2SQ7f6^4a~zjvV=sMi;bU!Uf5#I#y27d|dUonm8HGWx-G4ZA
zs#Jc&S!~mV6+IeBHuIJX1gGD=l_+VXUz*_mF_#rc#%Vbb%l$m*0T39VvXG+<A<L+!
znp-#@`Lf}Oc5^;C2CS=E#QO%&pw%vWS*Bj?{5GcV+B4B_o#>xDc^8pLrBf7=8QMq3
zO{8Q8rHyLEhM@|Xt3$cQs}{H5lvQ=uLJ7^2+NCaQKcQhsRY6hVvCQ2;2r*8TtjC<4
z7$}iN)Mg9c&-iBPYE!vfv_dDGRB-nBK6A0MazDtGVCxvn*9z_H&n!>^5RK$|bBKW?
z@GDV=hoIM5Y75Ussh+vNdtU(zywX?tOgbPej<OgdDzb3H;$J(iOz=}t*#UOzj2BPx
z0Ox0azr|TBw+9)TtVvYuXa*M&_DHl@ja*Oe@fFmgVwd_ExSuVh93KPV5$|WirKe|0
zMRgS+b=j**|Ie<Qb;Uh33;`~t5(IeM)@ZKVw|gsh76z3D0Vey!zvi~PnY8+FwX)dg
z=g)b1A2XmQ!GropEWqt>Qx61g>;^&qoY2n~FL_K>W}duW){B>-b?vC^+>}Ro$7a4;
z;AVy~TO0J(ecM=-;m`!kVQ{?L{r4{<JHTOfuHEA+FO8I<WIKG2Cj3b_YE&MXx{#c$
z)E%Os=Sqo@0Hi1V{mM}!`Q&k$Sj!6-*aBI}G=(`p*TTX9@tar(IJt@;%W1>|uWfHf
z6UDuvdQ+&1{XB>&6TJS0#Trz1<_rv$DJ}JHV(XcjLKbk!qIv!oowUw5;tL4RPoFOD
z{r1sxLoHF>VbVm;2rK@PQ&i+<W?XGJJR6L)XixK(ayi9&rk2P!qj{rTVRJBOuweY6
zXLfc>wnn~z<Q+dC)@PbmR#q6qh15uNL_|VK-AYSKu!LGeBeq=`y(Lq6zA0q-X5ogm
z65?R@-KFAvjxHg>0to}C5MIuVx$n}{mz0#{J?@^|NYVHx92~SW)rfRpjiM4CZj_6l
z7u^dF-9e2;Jh9_(2R=0k(o_P;ez8OOK`8iD&G}$FgltFX9lGEtpVA^1`DFGoh!9@6
zp}DrUUUPW!x0~&M6IYDh)1NimV-{_qF0X@G{EY?3RDR_p0>$UHy9Ej<?V7_DKqGP9
zb6GE!{IS%?%S&P!txrpvr>RjFwj|-GY~6xBpE}F6k@~gZ%ivF;7Xj#sjggu8j$ad%
z_#d3;v4G7U%Q=-~bOm7vUZ&D980<W>>vF7g3yim>b5?V{E*e)_Xt(NaTbt~(SKiBG
z)FEAt?u#msqk3bze2|!0;?Zg)in~pH6Zhc_$^b66AqcA;J`nu&O&~?Z!|lQH@}3BG
zWP2{55<R_$C?(}lN5-KTzO#o1&hS7P^q&rR;WG%=k8MdNJADbSt%iDAB^4F6Y=&_5
zit_xVB>6U^0~bhbz?!aFW2`shk@_vo>Ewu{gs#0SUvNy&?NIk{^A;4VQa;I1{y8+H
z3W{h}$+MNTtMA`BSQ}&(La+1<1mM7TkVw<B$%=o^S0o9{<;<M5ri~8^R~J)K`&n!`
zAeSyVS^u5fP1N7-zLZE@(VwR$`>l9NC09g`LPjHFCUomwkkGe9Rz@AcNkFI8t<t4K
zR$IIH4QB#Tzwqsye!GsgNH|@F%nVNOyTU1cYkw;>-cgQipPoC74f3o$!l#jvva`4E
zhZ7l%RP-BN-TTBo{KtFMhb@;vs35<H<vh1%RDQ=}*a;dmGkg-{Xu~eG;i^~~Le{J+
z4-Z6ey)iU|Q$VrW>d*HVZr{Gm&dyPa^Nf>b0fugv7cb#Xx3zbejh3x}4-k<EvkLdd
zgx7Q^&;X-}iGSJ6R1~x2&qzag9hvCprcO_@TS!~rEUxmC0TK~sBDXN;ovx35#O?DK
ze%JjPmpR{U79~5THs&Yeg|AP<ID^iWhOd4dy?u8^8M6hp<aMaH)RvDA-4Osx03zT)
zsbV9`3wvHp4xwCNWAD|=rEitxHNn2rS55&jie3O`8yI=Lx8<nps9svA^*u&fYRdpT
zk_i=$$47HDUBPpZBemph(EbLAJFB8Un5U(ao+@v#3UTM~1VN!2FutaymM(w0AJAti
zD6|=7lJmQ~u&e`L6{d>0?ku5WYQ%o6Q%U{-PIP9bnn9C}tf;B62`3tM<!PW_Eww?v
zRVv89Ku<s+V7w0bCD`HbLkQwU0x1x{QUCRDi;GufWO?@N0nU4w^z1^SfWTI9@dy=_
zz8EUk>S|`yidU_<@hyS3FXNNHOz8pi#?kQrFb>}`sn4=t#{0Hj1r3tt0t$%`sD1>h
zXE|N34m<n;eL>Za$j7RV%ipYY5uhc5AdhtVk@Sc%m8*due}}`Z7!6GXsigRY>6sZA
zjU;Yj`!%W8>5CfiR?iRIh?8-&UActFW3mLHNuT>Ti@@Zsp}UBCF1yBuE+J{R{hE}n
z(t7;3{O`<`pnqkyNRn^s)Vj<~w2&j9mm(%A+Wqe5qn9rqD@DWIM4!q0#9}SAy$&g_
z9Q=w35-uV*jPIUH9Yw3h@I$@V_xB+>q|Z71OL6fI#JPYP4nj+AkO`pSROEX>u-<~}
z|EcUPfU4Tt_wS`5iijX6DoBVR4JzHD(xEhngmg<Iji?|ZAT8Z3DV-_}(j_g@U7L>m
zzfsTmop_#ip4V~4Gv~}4+-vRiU3Xm9=i=m)jJS0Ixqr8#Y->4uP3U&aP}K#trzIP;
znm2(Th<s|vRH}=3I~Q&wR0gGRTpj8t5C)N0HI#CC{8$4Q_gut1yqdRR>PO;@m%UU)
zu57uDe9oISo$)S1GX@<U^$;NRj91;vj4AlpFEOD^y$pX<cSFN_qoepo!eGA!a7tRx
z8W96(IhHhi;^L*y6>(9lwA2<3=bN-(s-bDvT0vpB`I7JuT_KifV*rj$Q>M~WBqy(s
zU18wJNPy&6_iuHti^J%iB3G2oa8pj(@|0_?a$;OoR#K)^LI=?ZuASX7HUb(My(v9I
z!|V&e*K`N+XL0(7K?;m}m2+ilw9PrljkYV9M8NjL8145<+%CbP!B)kO>3Eo!Tvd-r
zR78D)!DshZhJ)(TYl!lCSE)xff52ccDK+IYhfTmBTtBVV@GfpuJViO7iiF^zeU;=0
z+wnAchw0wTXg;(N=7WGhsB8OR9d)A>4ej&^clYbdqv#n(fGmvS$w*E88h*=aH=njE
zMM?#|&RriZ4Sl;=PeaRxAuB}4S;fn50PX|_jGV2xqcU1;WJ?<)I&LN9!gQTLj^N(C
zd;Bpm(a~mldg`srfyAi>xNp{|;D~fyFipF?U;p*^9d(y1Z*O|BfTHE0x!D=zci9U%
zYp^GQaUlX>F@|O%g+knFAJ?e1l+z}*6&^iuSex)VM@RzUBEfGf-uTi=eLXJNUg7Kv
zKr-Z0ZBIc~VOC;dVtTp}U|=hThmU>wR#VgYWp1&oh^$rTu{=N(<P{t^Hzl2;&mh^I
z>X3;7%T3r4O-{>z2#)0z`z%b`1@W}&TrQEp;jX0gm0nRsW)?Mek^}5`PJ4<^9)k@9
zj#mIEf&bfQTTe$)GC#h!`<oGft-nDEq(~s7K#o=(l;QUK8(vk4VE16rx3?&n!w@Vk
zE30yj7}mrkA>zMFQFQiVl=lrFLJ_k^riWf(sYpH55+0q6%D2XrfDI>~J2@{eZ!m{;
z8I$e!4M{*|Uu928Nw}_%181Ql${dAzRvvD!oE-7`aFHJYeo#%plP*1deSLTGI$vTm
zH(ylg0qs_#l-au3u6=g=;RZRo?O(Aysm(_@kWjrnh8)OhT~15-RmfBCjpCLu?k}}j
zW+L?3ae@KJmY#HkB3dyvf&<)auBe&wLca>KM{9?!+5DuC^o-#{ei?PGTI7w;j|Rsg
zmXXAaT;=EkOnmG3w{4+}hx#`!OJXLZix1`P!2Aq^Fe|I8QLOe&i>e&D`bUY6UEQ7X
z3NlFqC{kZOKo6%G%Qt$7-Sard<M899GR-a<h5Oti%hnjWrXsWca?LRc%s>^{uY)-*
ze2Dc-yG1{4Gr+*cY!}))idxZi1@rnxf~tS1Qfb9aVF3h?>^5Tdc7u1Jq1h7=G7+!H
zXT1Cf$}5h(J!jsWaphUdh-cJJ&>q@D9oQf2g6=Ghfcnc+3kk2z>-g+s&VPtfY3b;x
zww)@ELJ!3P6DdoK^$$RA7{v>V;J0JdB5)+!7GU;+e63Fq4fUQ*GK9`sxZR<oOkZCw
zQ|+SzOq#)>e|yH7JwmFYl)a%%qw#)jE^HZhPhz|USwPS`C<x+14q~`$ESL6ofWpug
z%_}8le<LFt;tdY-TuxNIjR{}-`mN@^jC1hn4vc;yicG-Xk&%@?@KHaMq14;;@e1w=
z?&-6@>r^X!cIy|!WIX$}w5Vv#a>w|m&npAr0}y&4w|v8&czZ{^{8>j|msMYM1t2Je
zgoGUJ7BLY4S62~zeQv?&ocU2F{-@H)A9q8+3*oSyeEz<m%h9KYl>+ErA@VU;XAD?P
zaaLczR@>09XJO$N!nEg-<=%aRvW7U~y~%K;+T8#+==gW&V+F2k7&!bz^c%N3X3&-V
z{arg2!HqMozCiUUyG06i{=eZ=UjERpf}%3Mc*1V~+xyt>?<~ga<*3#DNcrTury}^@
zL$qzCYT@?Ijvo<QX?iK30DQVzm{iaX(Z5kX7og&OLEqE|8#F>@HQ|I~V&ZMl{KG>9
zqHi$KodFPxySzA1ky4n<PB&Kxp^x(NCJ_2qJ$XoUUXZacNY&#4yJzNzrUSd}LNQ43
zT~V<<fq@?rvwU~C=b4@N@lVq+A6h(VDYb#&Tcem4#*4!-Q9*!P%j~@2v`!uSp)7^t
z<H2!u(l%*(oAYDg6sT<hn+F3e;NB;urWWKE9Fam_de9AgZyg>aATIZUVs>*jZ~`J|
z8=@P5viqpm;MRsEt<t3;KpaK#SrH)vg~oPM3(FL_H`#A&_Gw&C^?^L&?n*UBcWIO1
zQWB)jZx1}2jO;grZLyZ2g1hUVV+=FnQoE;**(_P;x31oi(FoxoZ89bPH|FZ>QCj`K
z^u%cybz~%R7oxKE3D4*Ls1KG@2u3Lx!aiWM+;kA^;S-o6O6DGnU~Ap8?5ZCLp{q{T
zv0aWHK0Nju_Ks7l&rabkY;CSWutQq>y-&*^Zqg_=pI+jF;^hW4wM^&)?Ahs?=)=Fw
zUYN|7O>uAX(<oqL1If%|=pLG<nVaaFCVhXM{&yjO5yWM1Gt;xOWaQ<G4XYiDs0KVV
zT%u=TLEWcBi3b_?;iVc<?Oq^r&U{oMtSUl%YwQ+VUahdmKh?$;bqfJ+&>SMgQlSUS
z>q>_mqphVVgdG13P>?tW2SucO)RdGQf7DnQ)iH5op-}nY!OGe(BzYn2=i^i1IPDu>
z3>6BHa4s?K%=^cbSba&A2rMmi*zc?&5eEhQ^rQq}BUFn_wQq)HYow6vue>B-`)pi1
zr6hBuiSH8B$6rmJ#zd9bN(hEW2K`hf@4&Ea|6r&jhQkC8TL{(=v+Z#w5c85b9=l~s
z1SVR@%0eIW4es5C+n)g&u)V$eitJ75^?B0zdX0*;l=!Yz>`SeV^ScWJAC<%dR#z>$
zqfARIe#So@cXxl`eov!H`Y8S};Ai=)GF@T0*EcB<I&hh2slTX7{PD(M_WAB{?Ry?L
zljbGzT*w`O1_uaXPbzAOi#G)eFlVb2`I4U&)SaxunGYVKlzW$*95PnrdjuM)dgseL
zQPbGv;xf`iHYcnhQoS6z+Xkj-Lno&p=Om87^P9{lw0QM0P3_^U%6HJP0x8s!jMq(Q
zmHAPj(+(U0kR#wG^c>Hq_H}P)5+uO@)aw`mx?Cq0+GyTuFAuP<EojVC3Ez{`9{v0R
zoTCsB-q9hpQ>NW<=|_7){R08?-IOk9z(FPjI;I>+yt|pi2^+ezy*|Vcvo`wCE=H6g
zgz>h>=_lyB=+Q_X*D;o@n;<r1EXYV+*;+OHR#hpTIHQ3wt~m;Rq@W52>JIwe-mY2u
zK#ayPa>k0+c`Z?Cd8BM}35GbZ=MQj@0`pUWR;JUzpWvXEu?kuL#|LT``LE=lzbmm@
z(dU1ohvItwJD|vTXUEmkH^iBDm*Z^ik`9kS!^-Ds;Z1W(y_;c(rg5Kc!-{`*Th93e
zhmBPPagTLo(fhzK6bHcMruBF#2?$==+h`tWZ#RcmjMUKk>M6Jef=WtT5{WgR<Lc9U
zj8jKJ%lG9v1rUioJ+Z1Z*gT2_xOx^H$#<57ox!AaDEVo2dea#J>tblt$X+q4ms;<>
z4U_7>SMN*GcIXlf1H#$DTP|N}Do&j{beL#BU12d~B|0bhi<cBfSa0yz`Bs>u+<IY1
zZ&VG1KSO)Nx9)E>tZO4tFxeHMz9d0E55^gd#N={OsW|qTf#J%$JbdKxSv^y=Vd%Y7
zja4GT!Z@6@ErOr*2T*~zEryKwClCk+9SPFY&j+Vc1?(0Fun?HakA%MoCw~o+_Lj9x
zsOwBGXQiZYK*T!;B6sX}(CzOP<V&q*Z#&n$&Av%&XJIqnlhGG{a@~M+0ItN5cbnq6
zg~{)`Qt%-p5?xkYqI8B%HkwLRS-JT*1V=>kNx?~U9EiAgVcY0I*3sVG-Nhklzl(uf
zmef;|YPr;u0@Au6du;3c0zm32yq8qj<^T}fRnFI+H7Tj7NYCJgLZNidWBFU*qDm7B
z0tG&&`N~7{J4p7+|3HF4-2SZ-JEr-cC^dQi2_p1w+)R?uKaeW&-~S^&NHbUYV}-Mc
zi76J*;+I$0MP?)L{UY}z6{ytdh<}%p!)|(ZoRn|iqhlyTGuZh7-2mH2TG}jW_9!Q(
zePxD_hm=DV#(jeeCdj`zFrc05{jZ`+A0lq+%Tdy<fvb8DD05tp3K320gt6P!@@NcY
zqZ*H0rPIz`nd8a~ilWNz!_uw@rql~KSVQqThmZN<^ZP3ExZq$mzxu&^7rv^uAlo*p
zaPYaxHMchD4Y`UTC+GeqTf&sHe-XxR8V;eKt&4;3YDM{F8gIWof4%YwRqBUgi$w@R
zB5*F(KLME){(jjpX=#QwQY|2S(T<OYbUG<hkI+z%AYVw#mFa*-B3L{Lpvh&fXmwp7
zD&#So&>0cLe?IDrSyX*!5s?w+fbC)^V4i4V`6+!FKZZoKu(Lc34GkS!sS4?3zmnYr
z^1U;GVt6klvPXa&2F}T8%O2I6rsLF-Um+Rj%vnyWxgfPrCi&QZ;aRrL11J*`5{8Ey
z<%m&pJ$tpym&);@!LI(TyoATjWNTx!(&b<PM5G3gJTDx<^Hg14N5{at9b}NeYZXZf
z3wj5&!)U%mb%2mJz(oxD`p+R+@sPd17w`^md2w=badHDX2gKacHg~C;7G%zcLpl7>
zq|k22muI-{cMLX5FyM21JuVi2xr+(bU6l&U$)L0@=*$q!{~$ULHEpDyI_ddR=qfj>
z_CR-qqiwS-9O~dZvi|YSVzPE}n(W!wmz~&8Zk~`^AH^=!7(z)qJG%fk{3(41v+LgG
z;=9lHj=IClY|rU0C$aYroM2L;|20y~^g~jbin==K_hq}ureLu^YWjZDDwhMOJ?`Ff
zzxU#Wyb1$f#cs1O+%W;W%@0-C53a-rn6Hovk@<P~s+SuGN?Kc2-y!w8boH|N6p9v>
zBm(U3t3A&_dCz_Q`Z!q;_?yFS>kc$s6kt#<ZXfw9SlF%A5s(Chma}uY>yCX?m20eE
z1iQR*$Lw1~V6-GIZkXii?&d};dveE|WBfyMz_NdNj`fQ=*Zrfia}Pb_utjq7nevq?
zN@flbLNMrpDte^KQRINuU-ZqJGY}`~Fuz+M>yno6(HQ&2UzgeaiCHNL+JDz6Aa~?-
z61}~#qohx{0fUHQc?9BPh8F?pl{)wxq#WLa3=q!&DoB7X{PcZ%`0y3}?Em)dp`)*#
zqr8$5Xc*uTR&(YPN^|%Z*TQG*F)R;`!^uTW=nY!pW3ee9Y#{Cv3;6IM^{tN=h<}m`
z|I|icAvl-NxJ101i<8Zfky}mIA91>l0AB@pcDTPIAT6R%@l55B>=^{u>GFC}k&$p3
zpL7=wpscRO^&m@i?R5(Z0O%lw+jJW3u4$02qC8b$eqMfV?n$>7r#9jI>8lUkTr;&T
z1nEq4xKVp@i)$P<5b||(Drp&L0Bq8_s45>D7Z+Dq$=}0nX=S-zUSVHb3!t9~>+xN-
z0Loge)iqrSczBTS@O&TaZgq9_jHg?jI!9EabNdi`=Z_b|Z4JZZRq`9M3c)=9AY~dZ
z6>toYJV_Ms{vzR2VB*VNpikZAy7hdQ@1O^IeR?*wLS+Uce+;PU?#8+8s%>KbW1V^o
z(b3gi+tG#RuC1~076*qbYULVY-qZ8y__Yn%0LoVs*AN3E<1;Uh2T-Q1ylJwOl9HOU
z>(6w=qKzZ4rb-LvYR-WDrH#!|e6hd(R`B#)O8h2UiFO0z)1*0Ub3{e)k@shXA={?l
z`GrIXKO}RG{ByB)sBOW;Q+Ae^KeAO5!YcO1*O#?cJ->>J(tSMIsy)<bD8PxH#Zx?!
z>V4h@A4_f41;D@KmYg4*oq&R}G*oW0zkMq`Qn%nCwTxnp27zaF+J_HqCSrz$&MoN~
z-atbb#B)jIw8<)s&09;bJaQ8KbJ=?J{cqGF<C1q9+DA&Vmg0Pj#v?Vg;IP$!j>e0z
zE{CIa?KeMy&kLY7U$laQLyjXpn0FEylOM!9puq>n_naLgBjem9{wkCo_9do6i?IZs
z1d8Rdjk(~^h`pJyYM3S>G(ZAS%627vPJS9B20%Knbm0oEraJ8|p5jpo6FRYvvTXKE
zI^-~EXQC8at5?Y2zoxO@*-64vSOozht5uNwA#mo(?_~sU^qRBLzYO;8&rpj62zNgi
z=mCMwLR|TW$7=V~3X?#cB`7EegRUcbztavl{5NpU5;Ces21jjlXdR-@`HOyi(#OFF
z=9an8+x#)rlMMqNkgyFyaAgK2`f+1}?oba%_d!;gxBtEa($UrX@#zn0rG*C6r4iKy
z^8-<7@pti9e?nTS)D5!Zl7?3vu(8*J&bVc)-GEhsJ_5${bD7U${@_<f3Lt+Sa<Z`{
z94Gie00C<ySDBFC(`or<);jVKvINiO@^FR3D=24F9;p<V_MJl<evY|cQ#J;42DrOB
zfrTHL`IejDpL@PwME!iOvW}J(Y`~+Wm9#UCxrG!;o2fI21;EJ7WVDjcJE?al^7p80
zU>XLyz&CkQDBt&xjMMx(7y|d|;X)gMs`w(|1w0HGVZ7LQlHN8Q3dH&o@c$|-u%V`M
zeC_8wKR@>QiPdTM_qV!^@&H>hvn|G{QA>;Gka8f<sAJI@=F$tk`LJUMYLt4HFQ))I
z#{Edyz|fcgV{B|p@Z{PNLVW0G*zA^@ySnQW!Flx@0{mm#E8Ickr{REes(sUHYm0ne
zSSX)Lv4J@^%yt3p5n90g#|;o<v`1F~_Y<B)cVWD<yYI%X*HTzaR15-jqeq#c&E*6Z
zw@lr1Ga(xEu{U1GC~Wl6FNr^^&F<=Q+*p#(Z*Sh-uV1x@iNxpj$x2G=Ss_{0myUv(
z-39y|>)qXecE=(Du*Bc3;R}5B<@oCA>d~MZTsXM_bUxK06ZIVP^Di11Xs1k-t4NXm
z;9%~5V^$JC5a@B1#1>V9mHQB7eE+_rBx=(8(uo%@NE}d3>$@8uEv1%@3<?W-Vx+}w
zQ)i7l)Xp#+%72Ani5{uIT3xl=vfo??1E?<O^F!QOxoEkrkNd=bgF?s--pj!B_jnFC
zzCE)cPK11>#&@_Zi5zU`#zx}eT3^O-BG^M$cfixLsMV+28VP1Y(~^HUq4BL>H7^t3
z|9~?A5Y9~38uRm&z}RSN<_z#GjFX;ki3LcJXy~hFI;!>9+`!2Trf1~SSk7Qkj1qMk
zkM+v;K<Wp!OCvZ4SiOJKbWul?BoR<n5HT6O2jLC+*hb%r%_RW@!Jcjz4{lK5mXTL}
zHv6L%hnPJiC}aq70qY7`E>;3O`ELWPW+v%q&auV$G6CN5!ct&SID%^*dG`+jvf^RS
z+28P3l%WA3`*6+$Jfri3AtCTLwW_|GOY{5eL$hHJ9=jW3xDj3eOLR;`>5@{(veup7
zeUvMSn#d@g#Id+GnJ?o~5&X{JEv94WfntQt^(@A3Z0+I2;={ipZ2zxE{a@j>|6_~V
zx3-QxN5Z)G4E6M~)^MY3`#Vt7`7Df0eAT>t>{D-6GkovUVxC+V-k2K~G)-&LDnVJ2
zv1<j1pr1c_^e6^z54yy*sH*lAefYFUj^<eWu@jgsEdd+hZ1m(eX95LUpIAXd8k><p
zzf~pTOR9uoraFs*GYuoCn>9m2`!LxIX<CB@f}grM=7U9L&|tZ)aZIIi?)q=5^P$F*
z{O^_TC6m+aRVC)7wfsd14RlP5OtgsXm!&~@nh@xn1X8EzG(gm<Cni56wq7*TP*D*(
zFk5c_m{`oYB=|P1q{Jqr)xu9JBEsW()q%rAk;bv_+l_(VgiK#s8h47sH{le=;ywrz
zV9LU0R?SN+FE2kC2&>i+I${f}^#bj^LRpvD>H43N0NQgDzi%WOi;)X$XJM6nK|YaJ
zqA({X<YaHRJfCB@yqelcM5y)^2>);d;^WfNQl7qBLt#O{MjmTH{y15!cv9yU`21b>
z3UxoG!=`;HR0D-`7OZlECE7e}R(pD=ndc`?LCD@jV$Hp=F?=tb?brT`g9QmJm%)BQ
zCm9TQoUu^$r&$J1m+>wT#iz&b!qNcdcJi_D!_wib{RK+QNJDeWaEWu-LrRPzO;=b%
zgt3vq?%pm4nIAfW5V>D~{GoWjty{P9ij^69O`ehwtE##Smzj@!0kg}6r1byPg8ql@
z?KtxG?OUgV12BbB1(foY*`=f;(GD<1NpInvJ<@QF1?%YRH+~_$v;d6?tg#<N^9q{g
z*v}UVFh%<LzPbHxTNd(f*epFr2X>B9L;gXQMc#87cDf0^0B1Fr*b^wOrv))Q%A@?1
zt1O~KEg;|u<}V=bv>X@26Y(R{$TN@;RaS;gkA)v82#PXPOshezFo(KQ6~i|^PWDOH
z>Ks7I;XsV5_JDfx>1g3a1QUl;OI!OW<eh3U2h7A=>r+zUoN<N8n4UL(y(}IQ6in=*
zXBx^L;FBpSDR7)+7+V#8#*1iv?AM!Ggy?PJ!@kae8{;<IY!?Ab37N@JW!7NHP-fn$
zoFfWnj8JeqwP$1OdIsxke!;xSU<nISTwHpNi1pcQfRNy^!vSt?csHFw0)kN1w%iV=
zqKnO;Rbaeo_<pU$hvp9kq!*LWk{KMl^AkUAQ?pXCa?XJEw3j_9)kg`volq2diA^^c
z>-1S}7nG>7fhb5`d@_YkN{4YnhyWWaCx|^>jk;{90lDU465S2D-I<Q-5ZwSS4*1<o
zc?HR5WovG3KCyq0^|<7=;hGMe`RDjrtW_-VBs$tIsuw@I^&)PC+g~+XeM)Z%{-wY{
zekr&J<YR~2ecqOjHV;P)e1h%Gr%&XEGFK6}O(;Q=57~ZO%mUf#Tez$SU6{xX>B##h
z9x02iY)OP}E(^pO7n!|j0$FQ5Yk^)%bj6w{nfvh*1{kI*-{a)VL9>b4Toad;K2+@y
z{WpuV*z_IbC76pU*TmgpNtOy$8!{~2FLwgAKwPyaWKHS}bT|5MEz=LkiXG7anHXu0
zJ$7*Wb$}cBW}^-TibkAIoh4PQV{U9dFc##GXzA_haYQeXY*2$zHz~7a{Niw>J);0~
zTStGAZWkUg0X?3)qJn~Mxr??YtoT8N3}q`e-APKZk#d*n*?VsxIGcfy_N3dw0t48_
zLd5Tk4=XAuLGB3zfI?^qq_j~|!G-_ucbuPag3K7-KhcC%R&qk(jo=eM{%O7Y!U4M}
z@+nq|7)oJ4As)Nq;6RjJ{H?c6v7;WUP<ZEpeF<dL5VY(J3vv@nS$xly*>Abm5lENV
z1L!4G)UW_{c0K@IHFC#H+Bc?%BfI!A;;?BAh2$G~x{Iez!a;HF9ACHs8wA9&BtMu8
zUJ(s&+E@e3dEDCF*MI3Rjf%iUsLUWO?ywKxU&!{$T68>}e|Y#nt-xVyhz-r?i}<gw
zW!l?)*4NwnWBb+bOU_-_=QSlLnFt1Ot3ETeQXALJ#KWbGTrZU;Px!^eQw5CYjf5zk
zxR^0Pi&dGsXi_gLmx1rCN{4gzx!W!%4MnJDj(Q}=v-7k4*#b!W>F`)sysdhQQsD;-
z<O#8km<kGhZYAM?h`%dW@z*GK8sRMu^MeVOkm|`d_bH6j9N=O}*&KX%%H&Bv$PTJd
z<de=#9hMcn3ld3g;jendD6NTn=m?40A)!yg6u{Z(aY^Qkn4uYmOVt;z#yw0;OG``D
zq{0_7F?kDq{IIYvFw*c8LiA%+X{jhp;@U~1dmQ%&#zf^ve;>dpuU-j#O!pvv-H}j_
znwh1c5|c7r)Q3dWdp%CdM%S)UH4S?F?D1lTFjIG26osc!*e?GrPj_`RoDKmiLr3TZ
za^ShS^sjIGe5JFHjde7;hez0zD|w7UcdrGeb*_ArU2D!VpIVnd1GpMsJcqSs{wXrs
zTppyiV*y%5a<VWUhFAcQ&*;)OPS*a?{Y3O-tL7<-OTY~{esB<O>8eU|0qk|qwkomH
z(0xm7Xea?gJ*aTNBB8{5EcR{Nv^kf2CGx4gJ*T`J(AlA@&D|wKcL7P)-fpl&e;M{J
zkB1!~_D`7s-;abtT|=Yl<Y3+O=(Fs!y+KmZfc>246QFf7m;c?RJ7fKS_Cn8N0qIn@
z+<qPzu$fV&O-&pj+McAWp$1=(CD&u*@=eX}ZqLyxC!d#XEo^MAntsOkyKY6N=rb`r
zB!NQ|g0wVp(|kp<Ki=!*_V$x%4+0FqlrH_Rs3S8He$-E0P#k+mg<$M0mn&$HT*(`z
z8mG2}@r1TKp&!^*7@!Ba@r}T=w7CAR4jKE;$$x{J)t)5v3yp{v1WOZg^JlaXG}y=7
z{;eef9oau+LGx8Z?Au9B*RK;GpfAX~E&rvXk(QUjy;0}?o}g0jL5^nlc>?HE@cL;?
zXILj_`E#y~EWYKKHDU<SuX>IVMKivx>y5s07Bld3YWnWr+TqJ$5x%G>p8lfRq2~y4
zbye)Xp!|S>q9P?#jq~hRYAPxqOrXo3obbT}u6~oobTTZhv!AZOb_`uS*rHlwXao%7
z)J`l1-1=yk%L_PFNiKyjtC!Er&z2asjX*jFMDA3S*+N*o=XCd#u48w6G7Bxb2=N<0
zu>g(Y@;?zu>c`Cg-G&6br22c-&ud7L5>hoAE$v|@*MtT!@g=n40irar`MM_7)>~P_
zTO^#y^77l{H-tr$l<YqX3kmzi+#KtB{krOn@!xCzHF!~T{#%E@Mz8T-=vcFX)W)ud
zR}fAUUp4)*Sr&cakqSFV1<USdzfjAj`TY|hv%ZI}%E?88`H^N+T|r1eLJnKU81r{`
zkc#!IbqJ6D6IfIef9D(fAI*amz>03<fsgk`tw7FIyh5B`k9<3tq8$B~Ad6?{zxD_J
zE#l&T>;nFKl17IO3P;t}kA(Y??W`V}okxNYx`aZJ2h`XZ(qI4vFCwKF@S11GW=x*w
zn!;~gQ*BKH)S+o%oWIO+Wwd<ktKfo6wLVmSs0Kw_SF+SH_%QbE(*sOlY7+{%KtOdS
zLQB9}aIjTvYk3HWc0SvSqC*Lu$DhY~$#9G;`l^*tOGeHtjZ9DZ5c?E8?aMkCu$iB@
zkx&9&7j~!Z*To`!{Pws<P_U2vaOg_)Kda9gwy}W?LM~c!`xW*jQLjXyxvbf;jdfh}
z<*w<DYR{C`$d+e;G7sM|@AdsilHjq;f3}Ol+L#8Jn^ep#ZIC1Z3O4NKllV7p#Ep!a
z^y_LT)0YJ<OD|f%e*(mgnBz`*PU^d~c>K#3CdQ{ZtQ*<~N=pG<rjcL#`#y?#bhz(n
z-408hGG)x<@e-m?iS^G-%k?sxsNew`xvw-+h+;2uOiHDwl)ZoWUDNi0GG|>~Dg=UW
zG^#Aa&amEM1F#n?AsC~dn39kbjZ{pxV98bX|3bOYY-s>nqRe73{2}H3%WuTX?5vy(
zb#--v^+myK3?Szt^tyjn2AE)`QY`{z%QF)}5RfDmfOCS8F;iUJ9_W-YIjXs$8004r
z|A&+r3;=3?wanmvZEM@!yyYj{QRW1fYp1owlUr~}`M39rG2F$ouXLrMAyPD3xnxdZ
zmmayyGNo#h<?zt0%D1pJ+-APBQ@x_4_3-77-^X^`l+pi>d<|SXEm~L!*Yz$vF+qhk
zth~e`I3i~<5X!l_uo!i>$5qc*?}C60YE8n^bUojUV5trx)3_s>?{-H$%;2t)M1eS6
z=Ng>iCos?IeC&tJq7s{ROoTzXct*@}4U{B^ChSgWAH)FcTw=#e;I>oE>+_>;@z2na
z@$GvKZoqqwrMw(?NQvY=K5zJK7gFWvNFX2!tTdaZs>d|jzh7f#XYXE?ykRx=qqbME
zmy3%FfC^06G)noZkQxbA<8u*^I@cEE<@F{+*L9e=YKHi7BCm5HklU#)8Nkl;A<YG^
zR}^tG2~Jpa^k$FC_@Df^Pqy5j=PDI_djljN6Qe1W&V0Q1WCCWK=+H9Nl9v?MXsM3f
zp(K<cp8xx{>*J~jv?h^Z!9r3E0xVe`$!(E4d&|>C>Bh*X{@Te;hZW!*1IjA6*|V~;
z<gKOruzwF1_@yNH|6!R2822f}gnjU9xthAVSvC66Pt!cwfXGm91oGGK(_F}ZW(K31
zh0<@s#V$U(UH{(EU?E!m-v*nFd;A2NhN~0B0^4E)9<)r-02V8PhuUPaiL9br&(O$#
z?#-heGvdy!o(N#NZ-Fc$QeYH>+dFmeTqSf}Ip$Jnb?;so4<~AIK2Ap)IwKItJGMnD
z)G_bbF4&0|M*b58FPzZ0ne(T|*A!fJd8gvwLZ1=3>o59PAg<>htDUhfLOa%xObQBr
zQJM>~WaqtJA)m%q-A|lojpV0F%P=nIi;^ZgSS-5KiXNe`Xav}-in6lsaMgnY!=E14
zI|tD}ZM&UAe?03mT{kPGS<Nc;=4R*h^>rsEry7p;n-oT8sjrQ{^kZK#c(So=F%&9V
z_or=hcY0#t0uWX3<d5HZwk>$vph;k)P>ly0+Z&dmtY^#ZwqU3VR>@_n5mjZC8vvdQ
z^IG`nU^AXehJ;;bdSiMjVjgZT)1jPbO0it!a;N@Iq9loWmJywYb`c5M@qk)O)sgU5
z9_s42#Y4|SJ$>N2IH)t0dn;Go;#ZHd@_3ch)@)Q=RY@XX;>Z+(M^B&e_AAFLAoH3}
zZ!$_qNFe3_poV!K@<~>kg!=-344Bo6Z5QV%olhORBcxFKF(xI^S5m@tqbr=v(k`O`
zMh9ulgaU^-<)x*&F%bsY>dSefQ<GD4jbEs1*nU~~lA?`)DEX&f4)}7-_A%93x#7g1
z$N2dTbq>#FMZ4k=q@_2_0TwdUY&eEbEcv*P)YmiE>tbt9v$?xEd*eC`CY@ZoQ$BzB
z^5t6%rP2BgmD5U5^!qDwmC3x;e7_t{&ZxHzDk@htykqkMn#WvRxVawtK78nk8&;a}
z;U%|{+C!KVF*&9Ro>!3tw`s{$-vCBe9;atRFINTjw-XYxmQZPOu!foMT8}H9qqMPh
z&(gu|Ve27FdUNDc=yfem%y$uUmDTzI99%?mev<@7O7D7(Uz}MSkJ#R}u(S*a7^b5r
zYmx&KQIMA1au)|X_al2k_^5(M-0}74uW^Tvl?V?i40Re(8|?a@7T8$VDEyeJ``sAb
zjhXUxMaO{0sIc%H0$V&?y>c_{?LPNSsPkW)d_`tD3Y;QnZ9+vGAu;DcH0Ox<rn<T!
z7(t{DAhlZ0Aax-F57gh#XYxJ6+mqqkc*on__LLrRgP;$=z{SFn*$Z!8M&brq!P;e#
zI5c>bji1*V$jq6UNop}}xDr>0Zn(AR$yJ$i4T&d>s*wLQdTg$*j+AJZ75Qa@L}EI$
zN|bWSNz@G#aIWdZsp+Y<h4Yd7lxezl#)BnRVq%avN{!vv(hM%TMG%oe`&VS#$3E?R
z&R6jFEG>H+)<ykSv();Q0L|te8x|g}!2WDH5)j0?2}<pTM#c(?A+CCffHwenrL9_N
z!`;%s!Hm>&%9?tPAqIY?tZdIyo-Odos*}u?Nf4p$pAItY@+ITx-A=QC41=R?<fr`F
zh8qDM_Ftg5C0-ELQIwP4-PLvd^q1;txt+Sz)m4zS!ZryE(Hgn&AEQW+6^SF=JaQ+c
z3zYLs?1f9+Vxv&U!;*o><sdF0Q101%jKG~f`c=?TMwb!=L`5;;dnX1EZc}*tJ*ad)
zrSv(-Ktw6iX|fIahYO#M9uAik;oKvwsq-ab5r$`~rn|eRKxw>L?AN8ahI*9`(<OCh
z-tL|rEF!_1Gxt3Fz)tGFWu{*)rlg(AyN5$La~~Wns{3u4H(@!b&{Llg)uX&Hm)@p!
zTq>ml!s&FDpQUsdzya&S>9n2D0;w?n7{j0Zi+Yp`t}t>F^rO_q#IPub*dM=5VD?&A
zbR;vXay>SKiHf`UG2}>&{5f6-thiIbD*^ZfSlntIX8JARrhi6bYQBqcg}A}x>DGht
zCohy~N*y-DVZIbJftv6l7^$q>tFAWAFb2%jB?1DRqfN*O%yUOj9Z0ian+(hK!b0#c
z_*O04i;Dch!c+uh#-56}1TR96(osQe;97C`9TI)LLzxmf+$fWPAu<(O@M-)q^wnU}
z8p?U1EoRsV!ZZ^T^ycoTk|%LK0{AW~cd!c|{6rKE%ex%=1U%69Gc#a$KXyk+DOBUp
zz>f+EB_%(N-J<w1r&(Fzd)0h<n@V08E_-Yg9H*B5z;9Om;**|41^<z-_s6svZx{cv
zG6vciLvVZN6ze$w*TbzZ%<EY%nRQ@14tPQkA6nR0IIr)r^)mc%+Oa_?@64xGW*xOS
zeow#M`;E8V;x}pM`fGm_QrXB~Q}0AH&y)Y20^H3c{Zo)~`XAb}t<^oW&3vLV&F`<#
z%ZKzYw*Dufns)v+DRsXxj+s9D`T&Va!$KCjHUrOXgZ%newN*7#XN-(qEG$;$8fe5E
zcRx)2@O8P)8JRIQ`Z3%h{7RY`->bKtYFhqGIB}VeuWe1vA<n%`UNNqK@XhQ?sBbFj
zCxm^2-UWou&7r!dQ_fxezAk=PRZtlA<XV&?s=d84oX7BuD2;rj#q#qLGiQY-3IIjy
zihe3g6j8)iYTQSiXj;Byq7d?^x^0g5if=Ug#&-{?JiNBCY93`(l7YMpi3`$p`k}cW
z^%%CW9|pLEY;e_H+&;*Q-`}_LybeF<$*)UyXjd??uc7veEC1f$m8eYYmr$Okl!&=n
zeVkQ#ifv@nw_y=xHfY!u7&P3NRu(9ihC|53r4y`#xcL&zI~I|$WRElwoUXUaQ=S{T
zK4UwPl0=ysyDbRlVMohr*{hhWh+)@p)%Ia}rPh~ZkWN$Q=-74|r@v5ZvK}k8BeAo0
z|FBl0Xc@!J`BE!~uJ(0^rY$ee4k;WAW$M6ksK(Q_TF+MDoNbfaLg$TXNZuisotejt
zx!Ia$H4Ps{aY*r(;?T(XktKX6jpd@rU}A!Rl>SG}$9EKZ;2$Mw6%f2orI~i&sxc9U
zd{)QHb5F%YG2II1=4#{9d7tAU_-oh|3nu(rRLtVyzEz@6FW)Z9@OjYJYPBTn$!Tqx
zon~=pQ<uC=y@|Q&cxpq9Ec9Fb*Tj^#&wFP+PY67{jYZL3+5GyPslp8E<(K#vGOCc*
z8JAfHD#Wz2^eQtyr0w+Porq|>YPl(^VuOQl=68&;<Se%)>XX%`irOgj^swmnE{sax
zVIU?v&ObtTuq&ooJB+p8@2_<DNAVns%?KP%O_Kv5o)9g1W@%W+eh1OiG-2%@ZEN`Y
zA-QyFtub)bUj~V7(vm`L<Sv*<Tz^em72{Xx^5TtdcTgkW5Cn)kkv6O8HSKP(EqQL)
z=;K|h7%?&3H!>hj=g(VehaM;yJ*KI){k-Pesm&}Vjy4VR+3Ch+X5MleWB9&>+buL`
zeH>7Iba85m<XcW|Uio9O$8yI{SpVE}T<5XVuXH$x1nxUK;#?+A3w-;YRfYjSa|Jh%
zofNfrc}wsp>rY(k$LxtyRQOm?(lJE%-Hln;Rf!YMipc>1XaF^`oWiYeS#{qHOiRyO
ztXX|$L13~_)^-+KK}Ny(DhYkrm8I^Ywjei%60BTp$VPH(c5s1SaRQ0@yrNc~`l7&w
zHv1X6ZN7PDKY7|)IH5(g>;}_qx~VBC<b9c~jh3WI_XaUT;Xt^vBNq2H!qd82=!;a}
zS?0mz&c>u%avr`=ZK`G5OD+=!#(nQ4Z=8$$bx<7O8baH)wc&`Ybeu(t$s8=&X*22H
zuP+pdL7UQ6hDNn`U7EF#r9~3gcOIqF^>RO&P0}TuN?zT01-tHZa<#9klU?8xpprF|
zPQpWaQI@`K%-jcaQdxHpgzNY*1jBZDO`UvbciSrmDw<@Q!e|}s5_(!H4VwLtq<Y*e
zU-$y{i6y<dD<i-XNEE;uw3a7Mao+VkxtM5LY;vLSU3NZuNd*O23UH~&rUhwyjo_kQ
zKz&}}U{>!mU`5)yd!yOxmON^`*GDRrIV@i)DbMWIN*p8C?9b!5#9TX6s%v0WYSNti
zRmrgY%a>a$$#DG@F)(sVBA1Iy8eI;i)Pz06!0r#k5lqZL%2_V0XCe{Ow0;DWm~`TC
zL&cS0ykVI6WGCFaIU{vd*>wR%xHEwz*v_I+c|lRJ7_~l>uhU=HMSYp)O5#gvbE^a0
zu^&SV3s<b}dis#r?P?as7w?BMc|iin(xd>KT6?f@G%0+brZ(K@e5#m;XqGk&rw`3j
zT(WtEcV_yecg@{6bSG8xb&t`?XK9{}PYJBn$@WiNru&*i%u#NiwxZ(a9{Jc=JE4zR
z-N3*Qi;nK>+qd>7VeP>GK=Ki%^NoVb>V+0gIzdDcraiSw36ooeo0}0K&M^^Di$yxQ
zd+$EGM)UT3a1sLj#!6_69I3!kXSJZLs)$;48ot*h=@`S_%8JndA@qB<whzoDV9U^)
z6qbMVSW+$S@-0J(Ki)sPqicZiDRR6sx}M<4KOH>c97>PPI+0ga)KM0);kzrRtXjml
zIy)A3@YI)#tttMXk~%Hs>Tr=F3Desv!UvagdXU)oQcR??sj1S>-g;I_QA_X^BHs;>
zIPC`4imtb%=#MlwoI*|vgim6<tXEGQwvSl9ncZGn>_hZI(DXxAdLA$L(zZ8hwCi^I
zJowEYc-x<7easuNne1gaZr&pz6}C^w<ojq&g9<-|SK4X)2Nm-08vAgWZn=Zl@PbdN
z#m9<ZHt)>jr#h*|vFPuuBnVAIqskAqvV7QtlyY+oSGo95iU|to-n9;@@Mb}}yXB4N
z4EoDT(({zJ4$JShtWRP#C2=y<mnj{U^c2s(Tgd0D!PYLegPeLVGxqMo75-4lMmm3G
zADH^S&CeeKIj!hh-+~;4;-GukQpES<x*9V25ONmA%>3eSoxO@3b!t&jq{DvjtqP#L
z0<xuw40=R)V1VDpI~&HZ_+EkIKZ1jto|t7l2PK^USexg$fwB^am7kcNVlOg;4;B^}
z3^JuISy|NCK~@fpB%xX*>F97a4wHuB+^WU>pi|2NA1fyaLx<djem(pJx8G(HNVPAD
zs<A05`<Y>1EqnU<LGtla?<4pc7*lu5uUAwgJG^raT@5vxP&l(&z&6}F>fAX=JVDKO
z&K0e;RPc4T`ON;D(e{d_D{8NbFISS!I7VLn{_sx9OnREEH$=XCNGC$_<rQjd;0%%P
z?Y?p8qO0XK6fuAKa{Dy}g&&t1A4o6pJdDA7SJ~EJR2=3J5)vC9uv?LpEX+WdVQgg4
zU7{8Dj&fe<N<+w!Rq;^&`cwcuJ~AU&7wGAS#Bx@B@5>zA^V`)ER&MZ4q^{pwI3J|c
zfD;rU?OR$pRLq)R@RAH)QAMVs^-DazzfA#ousEd+LQW+%SAzN3^CX($xm(L59UTFC
zs03oVc<9_6LzkXD_bB`g`;>XHnlNgnrwY_1f&^F?@>mlUnc2p<cT%(BR>pU@xXzhn
z*}@t`y-d$|G)hT12+f~T5=^tZA}uaU5!dzlK|^14*;d`x_x}4Ug7$`n-qu-18DT#H
zRC9h^{5ZuaXoB{N*cdOok+Y|w`jIZjtby+AQ(#9g>l!_q>d`1_aU?3VSAQI?N{hJh
zJhf6R<(f-NyXaLDgEza^)YNd5tH`$5IDHaBv4?&CV}81@h>y@gR*Ni55qT{RQs~k%
z1X5^Hx6o|ovNJObxAlco!U{YboHtTabH068>y#B!iy{iWcl|W2*m1W?|0lXC_QA@J
z6(14h+Nj#@&iifdz+(I3%X*`$n;6Ue2`SQCg^t<rpX%tgbuGu!zlXxT0ah3?M?|D6
zo|64|W0xVP<Z;}OWeXZ+#>}(~m8tQ;pi5*S!67622Y$rUP?q1x6>ZpE`QQnHxB$OP
zQ&aF#8r*ouQ)+DYsWXW@slp}tjD|;m6HW~0n2M8A3gou)*gAzmtce9M-T(f@g&k4V
ztx%p6eI=ZW7rA}dQ=?Rb`!$xIf9&mTV%VD}v{qCRx^zO#6c9w5#0BSjdxKwR@H5J)
zstwZ?Dt7xp@%NG6aeC45`Z>q+%#BaBdS*9-{ke7u67M%|4x={A{K(i^2jY{&7bXK!
zNIvjf2@iwaP5U*&Hd&+e3<(hCT3I^cUOkPNzV!(;9g!TCcipAQ8eKVml!k+{_^5A_
zQ;3O_k@8iphr^^Sj%I6-N0vh20!?Uqada&;iiH2IaDqkgoUij9WqmDeqmZX`6lx$J
zEHoTNeQh!H10N#W+8p<?wqga5PdyJ~Dm0@9Jo^oUl}(0vuOh(kIeUWuiLR?HL#-#`
zcR7B)Gbi^{J=w<27ThIs;o)i|B*BzvH<^_rR6SetLW6?{Cnr_<Z4_P58VrO*w{FEX
zh14n<D^k%lDXC&(r|>U9Y0g_|SJgMfbvDVbo#WQAmh@kTzrg1&8ZM;VE?0fie-{iK
z1rrD8gBN@QhN&#4xxEFeZkg%CJ>)%@Ta;t&!uj@r;QFasSOebphaSb0M%wIwje57e
ziNCIpT_TKf*l|E>J~$p5pR6a=eXlXa<%&A)y#rE@jqGbqK}ye-)mTW^4;k5fnsub}
zb9M0Z-w*9et%@}jr4X=YosoMrqNOC;w6p)(bkK>gRUaZjUI~fVb_-CsXdcvsS$z?f
zj;0|z?J`&y=uo%UG|NKZ?5wyvctY0DNVap3<0<leV|_a7fUj?9R2BTgvhr1@N}j)I
z@R{%QA_zR#S6b>nNoha-rS@xRcF@8#aCl&7kOdY_!4KK%%RYrb&*3R``4qy+o$Ihn
zN-K(MI<h-A+jTFZNE+wN<^4qiEU}Tp@qq2>P?5T!;>im0LtmB@x~3-EtjB8aLqo&g
z8*~;fc1jT&veJBQ=E<|-Kq&7E_1i9QHpiK(%Z!cB7%-}-35lg=X8F4ZIiS(r)DNO~
zcL^sZkWX)-27B1p6yz0VF)A|T+TzRieI!DT-<cb~@Au?hQ4BvVp;xO1ne=O4Geti3
z)=MuMBcVk47VX@lpuil%+vpU`?l8~|{ZKLnrnQ~LjTsEjmZXu*DmqH}hw2?D4AzQY
z&sMN^wSq>4D+dUda*$62Rr+`3QE3OK($JP{Y~B1HqJ9r=4*J>|OiJ1IRg1FNT+zI{
z+`OK+%bKP(2F;2LXa3k%`;q_m829<VwxxeI*<b)8`=h$J`_JySetTva9}@@r<nNI-
mo{{d#;h#RP|5YPbg+9UV#)mVZ!WxReFEJtM`&oB2-TxoW7^s^7

literal 0
HcmV?d00001


From 437965aa3ee9d99799039243fc63c6dae3c03591 Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 03:16:26 +0000
Subject: [PATCH 391/395] Add libcu++ build job.

---
 .github/workflows/build-and-test.yml |  2 +-
 .github/workflows/pr.yml             |  1 +
 ci/build_libcudacxx.sh               | 21 +++++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100755 ci/build_libcudacxx.sh

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a1c0c5ab121..1269ee11467 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -59,7 +59,7 @@ jobs:
                 echo "Alternatively, for a more convenient, interactive environment to reproduce the issue, you can launch a devcontainer in vscode:"
                 echo "git clone --branch $GITHUB_REF_NAME --single-branch --recurse-submodules https://github.com/$GITHUB_REPOSITORY.git && cd $(echo $GITHUB_REPOSITORY | cut -d'/' -f2) && git checkout $GITHUB_SHA"
                 echo ".devcontainer/launch.sh ${{inputs.cuda_version}} ${{inputs.compiler}}${{inputs.compiler_version}}"
-                echo "Then, open a terminal inside vscode (ctrl+shift+`) and run:"
+                echo "Then, open a terminal inside vscode (ctrl+shift+\`) and run:"
                 echo "$cmd"
                 exit $exit_code
             fi
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index f61c9ff8341..101e2557bfe 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -63,6 +63,7 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      build_sript: "./ci/build_libcudacxx.sh"
       test_script: #"./ci/test_libcudacxx.sh" 
 
   # This job is the final job that runs after all other jobs and is used for branch protection status checks.
diff --git a/ci/build_libcudacxx.sh b/ci/build_libcudacxx.sh
new file mode 100755
index 00000000000..da0c7ad4d49
--- /dev/null
+++ b/ci/build_libcudacxx.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source "$(dirname "$0")/build_common.sh"
+
+cmake -S .. -B ../build \
+    -DCCCL_ENABLE_THRUST=OFF \
+    -DCCCL_ENABLE_LIBCUDACXX=ON \
+    -DCCCL_ENABLE_CUB=OFF \
+    -DCCCL_ENABLE_TESTING=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_CXX_COMPILER=${HOST_COMPILER} \
+    -DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS} \
+    -DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \
+    -DLIBCUDACXX_ENABLE_LIBCUDACXX_TESTS=ON \
+    -Dlibcudacxx_ENABLE_INSTALL_RULES=ON \
+    -DCUB_ENABLE_INSTALL_RULES=ON \
+    -DTHRUST_ENABLE_INSTALL_RULES=ON \
+    -G Ninja
+
+LIBCUDACXX_SITE_CONFIG="../build/libcudacxx/test/lit.site.cfg" lit -v --no-progress-bar --no-execute -Dcompute_archs=${GPU_ARCHS} -Dstd="c++${CXX_STANDARD}" ../libcudacxx/.upstream-tests/test
+

From c542f6fd9ec9ef8696daf1dd9bd3d5fc070481fa Mon Sep 17 00:00:00 2001
From: Jake Hemstad <jhemstad@nvidia.com>
Date: Thu, 22 Jun 2023 03:18:41 +0000
Subject: [PATCH 392/395] typo

---
 .github/workflows/pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 101e2557bfe..d70f94055ca 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -63,7 +63,7 @@ jobs:
         compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
-      build_sript: "./ci/build_libcudacxx.sh"
+      build_script: "./ci/build_libcudacxx.sh"
       test_script: #"./ci/test_libcudacxx.sh" 
 
   # This job is the final job that runs after all other jobs and is used for branch protection status checks.

From faf9cc964df13bcfc71064b00d1d7e92f48f4130 Mon Sep 17 00:00:00 2001
From: Michael Schellenberger Costa <miscco@nvidia.com>
Date: Thu, 22 Jun 2023 10:24:04 +0200
Subject: [PATCH 393/395] Update the libcu++ submodule

---
 libcudacxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcudacxx b/libcudacxx
index 7faca047a42..b03f8b60f89 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit 7faca047a421ec12844d446f5e297061f483bfb7
+Subproject commit b03f8b60f8992262d439ece514681974ee39041b

From f8e1511f493238992855ed5d3a97a2ce2c871d3d Mon Sep 17 00:00:00 2001
From: Michael Schellenberger Costa <miscco@nvidia.com>
Date: Thu, 22 Jun 2023 15:48:05 +0200
Subject: [PATCH 394/395] Add fix fo gcc-7

---
 libcudacxx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcudacxx b/libcudacxx
index b03f8b60f89..2070f2efd8e 160000
--- a/libcudacxx
+++ b/libcudacxx
@@ -1 +1 @@
-Subproject commit b03f8b60f8992262d439ece514681974ee39041b
+Subproject commit 2070f2efd8e93a5f952d4fa013b460437fe7bdab

From 2ac13971930f4c5be4366a3f7b7d33c6029407b3 Mon Sep 17 00:00:00 2001
From: Allison Vacanti <alliepiper16@gmail.com>
Date: Thu, 22 Jun 2023 13:38:40 -0400
Subject: [PATCH 395/395] Update CUB to prepare for repo unification.

---
 cub | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cub b/cub
index ab1c7562bce..b729edbf8e6 160000
--- a/cub
+++ b/cub
@@ -1 +1 @@
-Subproject commit ab1c7562bceb7c8b2123556c4b077abe12e728d0
+Subproject commit b729edbf8e6d43709e3ff21f9f78d6178b54fc60