From ee9571c91994335cf846b537d479ce5abff0e666 Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:24:44 -0800 Subject: [PATCH 1/6] CI & Release GH Action updates - To support sha2 builds on windows needed to change dependency for sha2 crate to not use asm feature on Windows. --- .github/workflows/ci.yml | 8 --- .github/workflows/release.yml | 103 +++++++++++++++++++++++++++------- data/Cargo.toml | 8 ++- 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad8a393e..2af37196 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,14 +36,6 @@ jobs: uses: dtolnay/rust-toolchain@1.79.0 with: components: clippy - - name: Install protoc compiler - run: | - sudo apt-get install unzip - cd $HOME - curl https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -L -o protoc.zip - unzip protoc.zip -d .protoc - rm protoc.zip - echo "PATH=$HOME/.protoc/bin:$PATH" >> $GITHUB_ENV - name: Lint run: | cargo clippy -r --verbose -- -D warnings # elevates warnings to errors diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cd26201..4b524ffb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,8 @@ on: # - main # - master tags: - - 'v20*' - pull_request: + - '*' + pull_request: workflow_dispatch: permissions: @@ -20,17 +20,23 @@ permissions: jobs: linux: - runs-on: - group: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - - runner: cpu-high + - runner: ubuntu-22.04 target: x86_64 - features: "" - - runner: cpu-high + - runner: ubuntu-22.04 + target: x86 + - runner: ubuntu-22.04 target: aarch64 features: "--features openssl_vendored" + - runner: ubuntu-22.04 + target: armv7 + # - runner: ubuntu-22.04 + # target: s390x + - runner: ubuntu-22.04 + target: ppc64le steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -44,6 +50,32 @@ jobs: sccache: 'true' manylinux: auto working-directory: hf_xet + before-script-linux: | + if command -v apt-get &> /dev/null; then + apt-get update && apt-get install libssl-dev libatomic-ops-dev -y + elif command -v yum &> /dev/null; then + yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y + else + echo "Neither apt-get nor yum is installed. Please install a package manager." + exit 1 + fi + - name: Build free-threaded wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release ${{ matrix.platform.features }} --out dist -i python3.13t + sccache: 'true' + manylinux: auto + working-directory: hf_xet + before-script-linux: | + if command -v apt-get &> /dev/null; then + apt-get update && apt-get install libssl-dev libatomic-ops-dev -y + elif command -v yum &> /dev/null; then + yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y + else + echo "Neither apt-get nor yum is installed. Please install a package manager." + exit 1 + fi - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -51,15 +83,19 @@ jobs: path: hf_xet/dist musllinux: - runs-on: - group: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - - runner: cpu-high + - runner: ubuntu-22.04 target: x86_64 - - runner: cpu-high + - runner: ubuntu-22.04 + target: x86 + - runner: ubuntu-22.04 target: aarch64 + features: "--features openssl_vendored" + - runner: ubuntu-22.04 + target: armv7 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -69,7 +105,15 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --features openssl_vendored --out dist + args: --release ${{ matrix.platform.features }} --out dist + sccache: 'true' + manylinux: musllinux_1_2 + working-directory: hf_xet + - name: Build free-threaded wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release ${{ matrix.platform.features }} --out dist -i python3.13t sccache: 'true' manylinux: musllinux_1_2 working-directory: hf_xet @@ -112,7 +156,7 @@ jobs: strategy: matrix: platform: - - runner: macos-12 + - runner: macos-13 target: x86_64 - runner: macos-14 target: aarch64 @@ -128,6 +172,13 @@ jobs: args: --release --out dist sccache: 'true' working-directory: hf_xet + - name: Build free-threaded wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist -i python3.13t + sccache: 'true' + working-directory: hf_xet - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -136,7 +187,7 @@ jobs: sdist: runs-on: - group: cpu-high + group: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build sdist @@ -153,13 +204,24 @@ jobs: release-pypi: name: Release PyPi - runs-on: - group: cpu-low - if: false #"startsWith(github.ref, 'refs/tags/')" - needs: [ linux, musllinux, windows, macos, sdist ] + runs-on: ubuntu-latest + if: false # ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [linux, musllinux, windows, macos, sdist] + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write steps: - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: 'wheels-*/*' - name: Publish to PyPI + if: ${{ startsWith(github.ref, 'refs/tags/') }} uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -169,11 +231,10 @@ jobs: release-github: name: Release GitHub - runs-on: - group: cpu-low + runs-on: ubuntu-latest permissions: contents: write - if: "startsWith(github.ref, 'refs/tags/')" + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} needs: [ linux, musllinux, windows, macos, sdist ] steps: - uses: actions/download-artifact@v4 diff --git a/data/Cargo.toml b/data/Cargo.toml index 232f96a8..e61caafd 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -44,7 +44,13 @@ prometheus = "0.13.0" # Other hashers for migration hashers = "1.0.1" -sha2 = { version = "0.10", features = ["asm"] } + +# Windows doesn't support assembly for compilation +[target.'cfg(not(target_os = "windows"))'.dependencies] +sha2 = { version = "0.10.8", features = ["asm"] } + +[target.'cfg(target_os = "windows")'.dependencies] +sha2 = { version = "0.10.8" } # Need to specify this as optional to allow the openssl/vendored option below openssl = { version = "0.10", features = [], optional = true } From acd8dd06c624b90254f080b5a43b977aea36e182 Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:27:00 -0800 Subject: [PATCH 2/6] typo --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b524ffb..e21b1e23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -186,8 +186,7 @@ jobs: path: hf_xet/dist sdist: - runs-on: - group: ubuntu-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build sdist From b5c36876f6b712076c9634aacd07ae7f8b99d99c Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:31:18 -0800 Subject: [PATCH 3/6] Fixed Cargo.toml --- Cargo.lock | 4 ++-- data/Cargo.toml | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 519466a6..f51191f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,9 +717,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] diff --git a/data/Cargo.toml b/data/Cargo.toml index e61caafd..83ff4a01 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -45,13 +45,6 @@ prometheus = "0.13.0" # Other hashers for migration hashers = "1.0.1" -# Windows doesn't support assembly for compilation -[target.'cfg(not(target_os = "windows"))'.dependencies] -sha2 = { version = "0.10.8", features = ["asm"] } - -[target.'cfg(target_os = "windows")'.dependencies] -sha2 = { version = "0.10.8" } - # Need to specify this as optional to allow the openssl/vendored option below openssl = { version = "0.10", features = [], optional = true } glob = "0.3.1" @@ -68,6 +61,13 @@ reqwest = { version = "0.11.4", features = ["json", "webpki-roots"] } [target.'cfg(not(macos))'.dependencies] reqwest = { version = "0.11.4", features = ["json"] } +# Windows doesn't support assembly for compilation +[target.'cfg(not(target_os = "windows"))'.dependencies] +sha2 = { version = "0.10.8", features = ["asm"] } + +[target.'cfg(target_os = "windows")'.dependencies] +sha2 = { version = "0.10.8" } + [dev-dependencies] [features] From da22fd62c5f39babb597fa0e848aa7a0c9a01c2f Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:38:36 -0800 Subject: [PATCH 4/6] Trim platforms & vendored ssl --- .github/workflows/release.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e21b1e23..b9f79c6b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,17 +26,10 @@ jobs: platform: - runner: ubuntu-22.04 target: x86_64 - - runner: ubuntu-22.04 - target: x86 + features: "--features openssl_vendored" - runner: ubuntu-22.04 target: aarch64 features: "--features openssl_vendored" - - runner: ubuntu-22.04 - target: armv7 - # - runner: ubuntu-22.04 - # target: s390x - - runner: ubuntu-22.04 - target: ppc64le steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -89,13 +82,10 @@ jobs: platform: - runner: ubuntu-22.04 target: x86_64 - - runner: ubuntu-22.04 - target: x86 + features: "--features openssl_vendored" - runner: ubuntu-22.04 target: aarch64 features: "--features openssl_vendored" - - runner: ubuntu-22.04 - target: armv7 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -130,8 +120,6 @@ jobs: platform: - runner: windows-latest target: x64 - - runner: windows-latest - target: x86 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From c2bd9f74beadcd9e871cab7574f72088e8b5566c Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:39:40 -0800 Subject: [PATCH 5/6] fix ci runner --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2af37196..fb338856 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,7 @@ concurrency: jobs: fmt: name: Rustfmt - runs-on: - group: cpu-low + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -27,8 +26,7 @@ jobs: cargo fmt --manifest-path ./hf_xet/Cargo.toml --all -- --check build_and_test: - runs-on: - group: cpu-high + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 From 8b3a7ace883e5cd2afe7fa860e114499fdbd9370 Mon Sep 17 00:00:00 2001 From: Rajat Arya Date: Fri, 10 Jan 2025 11:47:01 -0800 Subject: [PATCH 6/6] Removing free thread builds --- .github/workflows/release.yml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9f79c6b..77095a84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,23 +52,6 @@ jobs: echo "Neither apt-get nor yum is installed. Please install a package manager." exit 1 fi - - name: Build free-threaded wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release ${{ matrix.platform.features }} --out dist -i python3.13t - sccache: 'true' - manylinux: auto - working-directory: hf_xet - before-script-linux: | - if command -v apt-get &> /dev/null; then - apt-get update && apt-get install libssl-dev libatomic-ops-dev -y - elif command -v yum &> /dev/null; then - yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y - else - echo "Neither apt-get nor yum is installed. Please install a package manager." - exit 1 - fi - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -99,14 +82,6 @@ jobs: sccache: 'true' manylinux: musllinux_1_2 working-directory: hf_xet - - name: Build free-threaded wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release ${{ matrix.platform.features }} --out dist -i python3.13t - sccache: 'true' - manylinux: musllinux_1_2 - working-directory: hf_xet - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -160,13 +135,6 @@ jobs: args: --release --out dist sccache: 'true' working-directory: hf_xet - - name: Build free-threaded wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.13t - sccache: 'true' - working-directory: hf_xet - name: Upload wheels uses: actions/upload-artifact@v4 with: