diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad8a393e..fb338856 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,7 @@ concurrency: jobs: fmt: name: Rustfmt - runs-on: - group: cpu-low + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable @@ -27,8 +26,7 @@ jobs: cargo fmt --manifest-path ./hf_xet/Cargo.toml --all -- --check build_and_test: - runs-on: - group: cpu-high + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 @@ -36,14 +34,6 @@ jobs: uses: dtolnay/rust-toolchain@1.79.0 with: components: clippy - - name: Install protoc compiler - run: | - sudo apt-get install unzip - cd $HOME - curl https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -L -o protoc.zip - unzip protoc.zip -d .protoc - rm protoc.zip - echo "PATH=$HOME/.protoc/bin:$PATH" >> $GITHUB_ENV - name: Lint run: | cargo clippy -r --verbose -- -D warnings # elevates warnings to errors diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4cd26201..77095a84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,8 @@ on: # - main # - master tags: - - 'v20*' - pull_request: + - '*' + pull_request: workflow_dispatch: permissions: @@ -20,15 +20,14 @@ permissions: jobs: linux: - runs-on: - group: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - - runner: cpu-high + - runner: ubuntu-22.04 target: x86_64 - features: "" - - runner: cpu-high + features: "--features openssl_vendored" + - runner: ubuntu-22.04 target: aarch64 features: "--features openssl_vendored" steps: @@ -44,6 +43,15 @@ jobs: sccache: 'true' manylinux: auto working-directory: hf_xet + before-script-linux: | + if command -v apt-get &> /dev/null; then + apt-get update && apt-get install libssl-dev libatomic-ops-dev -y + elif command -v yum &> /dev/null; then + yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y + else + echo "Neither apt-get nor yum is installed. Please install a package manager." + exit 1 + fi - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -51,15 +59,16 @@ jobs: path: hf_xet/dist musllinux: - runs-on: - group: ${{ matrix.platform.runner }} + runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - - runner: cpu-high + - runner: ubuntu-22.04 target: x86_64 - - runner: cpu-high + features: "--features openssl_vendored" + - runner: ubuntu-22.04 target: aarch64 + features: "--features openssl_vendored" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -69,7 +78,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --features openssl_vendored --out dist + args: --release ${{ matrix.platform.features }} --out dist sccache: 'true' manylinux: musllinux_1_2 working-directory: hf_xet @@ -86,8 +95,6 @@ jobs: platform: - runner: windows-latest target: x64 - - runner: windows-latest - target: x86 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -112,7 +119,7 @@ jobs: strategy: matrix: platform: - - runner: macos-12 + - runner: macos-13 target: x86_64 - runner: macos-14 target: aarch64 @@ -135,8 +142,7 @@ jobs: path: hf_xet/dist sdist: - runs-on: - group: cpu-high + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build sdist @@ -153,13 +159,24 @@ jobs: release-pypi: name: Release PyPi - runs-on: - group: cpu-low - if: false #"startsWith(github.ref, 'refs/tags/')" - needs: [ linux, musllinux, windows, macos, sdist ] + runs-on: ubuntu-latest + if: false # ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [linux, musllinux, windows, macos, sdist] + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write steps: - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: 'wheels-*/*' - name: Publish to PyPI + if: ${{ startsWith(github.ref, 'refs/tags/') }} uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -169,11 +186,10 @@ jobs: release-github: name: Release GitHub - runs-on: - group: cpu-low + runs-on: ubuntu-latest permissions: contents: write - if: "startsWith(github.ref, 'refs/tags/')" + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} needs: [ linux, musllinux, windows, macos, sdist ] steps: - uses: actions/download-artifact@v4 diff --git a/Cargo.lock b/Cargo.lock index 519466a6..f51191f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,9 +717,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] diff --git a/data/Cargo.toml b/data/Cargo.toml index 232f96a8..83ff4a01 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -44,7 +44,6 @@ prometheus = "0.13.0" # Other hashers for migration hashers = "1.0.1" -sha2 = { version = "0.10", features = ["asm"] } # Need to specify this as optional to allow the openssl/vendored option below openssl = { version = "0.10", features = [], optional = true } @@ -62,6 +61,13 @@ reqwest = { version = "0.11.4", features = ["json", "webpki-roots"] } [target.'cfg(not(macos))'.dependencies] reqwest = { version = "0.11.4", features = ["json"] } +# Windows doesn't support assembly for compilation +[target.'cfg(not(target_os = "windows"))'.dependencies] +sha2 = { version = "0.10.8", features = ["asm"] } + +[target.'cfg(target_os = "windows")'.dependencies] +sha2 = { version = "0.10.8" } + [dev-dependencies] [features]