CI & Release GH Action updates

.github/workflows/ci.yml

@@ -13,8 +13,7 @@ concurrency:
 jobs:
   fmt:
     name: Rustfmt
-    runs-on:
-      group: cpu-low
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -27,23 +26,14 @@ jobs:
           cargo fmt --manifest-path ./hf_xet/Cargo.toml --all -- --check
 
   build_and_test:
-    runs-on:
-      group: cpu-high
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Install Rust 1.79
         uses: dtolnay/[email protected]
         with:
           components: clippy
-      - name: Install protoc compiler
-        run: |
-          sudo apt-get install unzip
-          cd $HOME
-          curl https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -L -o protoc.zip
-          unzip protoc.zip -d .protoc
-          rm protoc.zip
-          echo "PATH=$HOME/.protoc/bin:$PATH" >> $GITHUB_ENV
       - name: Lint
         run: |
           cargo clippy -r --verbose -- -D warnings # elevates warnings to errors

.github/workflows/release.yml

@@ -11,24 +11,23 @@ on:
     #      - main
     #      - master
     tags:
-      - 'v20*'
-    pull_request:
+      - '*'
+  pull_request:
   workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
   linux:
-    runs-on:
-      group: ${{ matrix.platform.runner }}
+    runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
         platform:
-          - runner: cpu-high
+          - runner: ubuntu-22.04
             target: x86_64
-            features: ""
-          - runner: cpu-high
+            features: "--features openssl_vendored"
+          - runner: ubuntu-22.04
             target: aarch64
             features: "--features openssl_vendored"
     steps:
@@ -44,22 +43,32 @@ jobs:
           sccache: 'true'
           manylinux: auto
           working-directory: hf_xet
+          before-script-linux: |
+            if command -v apt-get &> /dev/null; then
+                apt-get update && apt-get install libssl-dev libatomic-ops-dev -y
+            elif command -v yum &> /dev/null; then
+                yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y
+            else
+                echo "Neither apt-get nor yum is installed. Please install a package manager."
+                exit 1
+            fi
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
           name: wheels-linux-${{ matrix.platform.target }}
           path: hf_xet/dist
 
   musllinux:
-    runs-on:
-      group: ${{ matrix.platform.runner }}
+    runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
         platform:
-          - runner: cpu-high
+          - runner: ubuntu-22.04
             target: x86_64
-          - runner: cpu-high
+            features: "--features openssl_vendored"
+          - runner: ubuntu-22.04
             target: aarch64
+            features: "--features openssl_vendored"
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -69,7 +78,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --features openssl_vendored --out dist
+          args: --release ${{ matrix.platform.features }} --out dist
           sccache: 'true'
           manylinux: musllinux_1_2
           working-directory: hf_xet
@@ -86,8 +95,6 @@ jobs:
         platform:
           - runner: windows-latest
             target: x64
-          - runner: windows-latest
-            target: x86
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -112,7 +119,7 @@ jobs:
     strategy:
       matrix:
         platform:
-          - runner: macos-12
+          - runner: macos-13
             target: x86_64
           - runner: macos-14
             target: aarch64
@@ -135,8 +142,7 @@ jobs:
           path: hf_xet/dist
 
   sdist:
-    runs-on:
-      group: cpu-high
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Build sdist
@@ -153,13 +159,24 @@ jobs:
 
   release-pypi:
     name: Release PyPi
-    runs-on:
-      group: cpu-low
-    if: false #"startsWith(github.ref, 'refs/tags/')"
-    needs: [ linux, musllinux, windows, macos, sdist ]
+    runs-on: ubuntu-latest
+    if: false # ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
+    needs: [linux, musllinux, windows, macos, sdist]
+    permissions:
+      # Use to sign the release artifacts
+      id-token: write
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
     steps:
       - uses: actions/download-artifact@v4
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: 'wheels-*/*'
       - name: Publish to PyPI
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
         uses: PyO3/maturin-action@v1
         env:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
@@ -169,11 +186,10 @@ jobs:
 
   release-github:
     name: Release GitHub
-    runs-on:
-      group: cpu-low
+    runs-on: ubuntu-latest
     permissions:
       contents: write
-    if: "startsWith(github.ref, 'refs/tags/')"
+    if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }}
     needs: [ linux, musllinux, windows, macos, sdist ]
     steps:
       - uses: actions/download-artifact@v4

data/Cargo.toml

@@ -44,7 +44,6 @@ prometheus = "0.13.0"
 
 # Other hashers for migration
 hashers = "1.0.1"
-sha2 = { version = "0.10", features = ["asm"] }
 
 # Need to specify this as optional to allow the openssl/vendored option below
 openssl = { version = "0.10", features = [], optional = true }
@@ -62,6 +61,13 @@ reqwest = { version = "0.11.4", features = ["json", "webpki-roots"] }
 [target.'cfg(not(macos))'.dependencies]
 reqwest = { version = "0.11.4", features = ["json"] }
 
+# Windows doesn't support assembly for compilation
+[target.'cfg(not(target_os = "windows"))'.dependencies]
+sha2 = { version = "0.10.8", features = ["asm"] }
+
+[target.'cfg(target_os = "windows")'.dependencies]
+sha2 = { version = "0.10.8" }
+
 [dev-dependencies]
 
 [features]