update unstable build

cloudhan · Aug 16, 2021 · 512790d · 512790d
1 parent 4cb831a
commit 512790d
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 9 deletions.
diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml
@@ -157,7 +157,7 @@ jobs:
         az storage blob upload-batch -d unstable -s "$env:GITHUB_WORKSPACE/jax/bazel-dist" --pattern '*.whl'
         rm -Recurse -Force "$env:GITHUB_WORKSPACE/jax/bazel-dist"
 
-  build-cuda112:
+  build-cuda111:
     runs-on: windows-2019
     env:
       AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
@@ -173,12 +173,12 @@ jobs:
     - name: Download Bazelisk
       run: curl -k -L https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe -o $env:BAZEL_PATH
 
-    - name: Install CUDA 11.2
+    - name: Install CUDA 11.1
       run: |
-        curl -k -L https://whls.blob.core.windows.net/ci-files/v11.2.7z -o cuda.7z
+        curl -k -L https://whls.blob.core.windows.net/ci-files/v11.1.7z -o cuda.7z
         7z x cuda.7z -o'D:/CUDA'
         rm cuda.7z
-        ls D:/CUDA/v11.2
+        ls D:/CUDA/v11.1
 
     - uses: actions/cache@v2
       with:
@@ -199,7 +199,7 @@ jobs:
     - name: py39 build whl and upload
       run: |
         cd "$env:GITHUB_WORKSPACE/jax"
-        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.2' -cuda_prefix 'D:/CUDA'
+        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.1' -cuda_prefix 'D:/CUDA'
         az storage blob upload-batch -d unstable -s "$env:GITHUB_WORKSPACE/jax/bazel-dist" --pattern '*.whl'
         rm -Recurse -Force "$env:GITHUB_WORKSPACE/jax/bazel-dist"
 
@@ -215,7 +215,7 @@ jobs:
     - name: py38 build whl and upload
       run: |
         cd "$env:GITHUB_WORKSPACE/jax"
-        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.2' -cuda_prefix 'D:/CUDA'
+        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.1' -cuda_prefix 'D:/CUDA'
         az storage blob upload-batch -d unstable -s "$env:GITHUB_WORKSPACE/jax/bazel-dist" --pattern '*.whl'
         rm -Recurse -Force "$env:GITHUB_WORKSPACE/jax/bazel-dist"
 
@@ -231,6 +231,23 @@ jobs:
     - name: py37 build whl and upload
       run: |
         cd "$env:GITHUB_WORKSPACE/jax"
-        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.2' -cuda_prefix 'D:/CUDA'
+        ../bazel-build-cuda.ps1 -bazel_path $env:BAZEL_PATH -cuda_version '11.1' -cuda_prefix 'D:/CUDA'
         az storage blob upload-batch -d unstable -s "$env:GITHUB_WORKSPACE/jax/bazel-dist" --pattern '*.whl'
         rm -Recurse -Force "$env:GITHUB_WORKSPACE/jax/bazel-dist"
+
+  update-index:
+    if: always()
+    needs: [build-cpu, build-cuda101, build-cuda111]
+    runs-on: windows-2019
+    env:
+      AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
+    steps:
+    - name: py39
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.9"
+    - name: update index.html
+      run: |
+        cd "$env:GITHUB_WORKSPACE/"
+        python ./update_index.py --url_mode relative unstable > index.html
+        az storage blob upload -c unstable -f index.html -n index.html
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+env.ps1
+*.html
diff --git a/bazel-build-cuda.ps1 b/bazel-build-cuda.ps1
@@ -6,7 +6,7 @@ param(
     # [String]$conda_env,
 
     [Parameter(Mandatory = $true)]
-    [ValidateSet('11.2', '10.1')]
+    [ValidateSet('11.2', '11.1', '10.1')]
     [String]$cuda_version,
 
     [Parameter(Mandatory = $false)]
@@ -22,7 +22,11 @@ $msys2_path = "C:\msys64\usr\bin"
 switch ($cuda_version) {
     '11.2' {
         $cuda_cap = '6.1,7.0,7.5,8.0,8.6'
-        $cudnn_version = '8.1.1'
+        $cudnn_version = '8.2.2'
+    }
+    '11.1' {
+        $cuda_cap = '6.1,7.0,7.5,8.0,8.6'
+        $cudnn_version = '8.2.2'
     }
     '10.1' {
         $cuda_cap = '6.1,7.0,7.5'

diff --git a/update_index.py b/update_index.py
@@ -0,0 +1,65 @@
+import os
+import sys
+import json
+import argparse
+import subprocess
+
+HEADER = """
+<!-- Generated by jax-windows-builder/update_index.py, do not update manually! -->
+<html>
+<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"></head>
+<body>
+""".strip("\n")
+
+FOOTER = "</body>\n</html>\n"
+
+ABSOLUTE_URL_PATTERN = "https://whls.blob.core.windows.net/{container}/{key}"
+
+RELATIVE_URL_PATTERN = "./{key}"
+
+
+if not os.environ.get("AZURE_STORAGE_CONNECTION_STRING"):
+  sys.stderr.write("Environment variable AZURE_STORAGE_CONNECTION_STRING is not properly set.\n")
+  sys.stderr.write("Abort!\n")
+  exit(-1)
+
+
+parser = argparse.ArgumentParser(description="Generates and/or update package indexes html file for pip.")
+parser.add_argument("container", choices=["unstable", "releases"])
+parser.add_argument("--url_mode", default="relative", choices=["absolute", "relative"])
+args = parser.parse_args()
+
+
+def entry(key):
+  if args.url_mode == "relative":
+    url_pattern = RELATIVE_URL_PATTERN
+  else:
+    url_pattern = ABSOLUTE_URL_PATTERN
+
+  link_href = url_pattern.format(key=key, container=args.container)
+  link_title = link_href
+  return f'<a href="{link_href}">{key}</a><br>'
+
+
+def get_entries():
+  entries = []
+  proc_args = ["az.cmd", "storage", "blob", "list", "--output=json", f"--container-name={args.container}"]
+  sys.stderr.write(f"Running command: {' '.join(proc_args)}\n")
+  json_str = subprocess.check_output(proc_args)
+  j = json.loads(json_str)
+  for item in j:
+    key = item["name"]
+    if key.endswith(".whl"):
+      entries.append(entry(key))
+  return entries
+
+
+def gen_index_html(entries):
+  html = [HEADER]
+  html.extend(entries)
+  html.append(FOOTER)
+  return "\n".join(html)
+
+
+html = gen_index_html(get_entries())
+print(html)