Justfile, CI and formatting config

.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "05:00"
+      timezone: "US/Pacific"
+    groups:
+      rust-dependencies:
+        patterns:
+          - "*"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "05:00"
+      timezone: "US/Pacific"

.github/workflows/rust-ci.yaml

@@ -0,0 +1,72 @@
+# This is mostly copied from the rust-analyzer repo
+# https://github.com/rust-lang/rust-analyzer/blob/12e7aa3132217cc6a6c1151d468be35d7b365999/.github/workflows/ci.yaml
+
+name: Rust CI
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, labeled]
+  push:
+    branches:
+      - main
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  CI: 1
+  RUST_BACKTRACE: short
+  RUSTFLAGS: "-D warnings -W unreachable-pub -W bare-trait-objects"
+  RUSTUP_MAX_RETRIES: 10
+
+jobs:
+  rust:
+    name: Rust CI
+    timeout-minutes: 20
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: ./
+    #permissions:
+      #contents: read
+      #actions: read
+      #pull-requests: read
+    env:
+      #CC: deny_c
+      RUST_CHANNEL: 'stable'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest-m]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install Just
+        run: sudo snap install --edge --classic just
+
+      - name: Install Rust toolchain
+        run: |
+          rustup update --no-self-update ${{ env.RUST_CHANNEL }}
+          rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src
+          rustup default ${{ env.RUST_CHANNEL }}
+
+      - name: Cache Dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "rust -> target"
+          key: ${{ env.RUST_CHANNEL }}
+
+      - name: Install cargo-deny
+        run: cargo install --locked cargo-deny
+
+      - name: Check
+        if: matrix.os == 'ubuntu-latest-m' || github.event_name == 'push'
+        env:
+          AWS_ACCESS_KEY_ID: minio123
+          AWS_SECRET_ACCESS_KEY: minio123
+          AWS_DEFAULT_REGION: "us-east-1"
+        run: |
+          just pre-commit

Justfile

@@ -0,0 +1,37 @@
+alias fmt := format
+alias pre := pre-commit
+
+# run all tests
+test *args='':
+  cargo test {{args}}
+
+# compile but don't run all tests
+compile-tests *args='':
+  cargo test --no-run {{args}}
+
+# build debug version
+build *args='':
+  cargo build {{args}}
+
+# build release version
+build-release *args='':
+  cargo build --release {{args}}
+
+# run clippy
+lint *args='':
+  cargo clippy --all-targets --all-features {{args}}
+
+# reformat all rust files
+format *args='':
+  cargo fmt --all {{args}}
+
+# reformat all nix files
+format-nix *args='':
+  alejandra .
+
+# run cargo deny to check dependencies
+check-deps *args='':
+  cargo deny --all-features check {{args}}
+
+# run all checks that CI actions will run
+pre-commit: (compile-tests "--locked") build (format "--check") lint test check-deps

deny.toml

@@ -0,0 +1,68 @@
+[graph]
+all-features = true
+
+[advisories]
+version = 2
+
+[licenses]
+# List of explicitly allowed licenses
+# See https://spdx.org/licenses/ for list of possible licenses
+# [possible values: any SPDX 3.11 short identifier (+ optional exception)].
+# This list was generated using:
+# https://github.com/earth-mover/arraylake/pull/1067/files
+allow = [
+  "MIT",
+  "Apache-2.0",
+  "BSD-2-Clause",
+  "BSD-3-Clause",
+  "MPL-2.0",
+  "ISC",
+  "OpenSSL",
+  "Unicode-DFS-2016",
+  "CC0-1.0",
+]
+# The confidence threshold for detecting a license from license text.
+# The higher the value, the more closely the license text must be to the
+# canonical license text of a valid SPDX license file.
+# [possible values: any between 0.0 and 1.0].
+confidence-threshold = 0.8
+# Allow 1 or more licenses on a per-crate basis, so that particular licenses
+# aren't accepted for every possible crate as with the normal allow list
+exceptions = [
+  # Each entry is the crate and version constraint, and its specific allow
+  # list
+  #{ allow = ["Zlib"], name = "adler32", version = "*" },
+]
+
+[licenses.private]
+ignore = true
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+license-files = [
+  # Each entry is a crate relative path, and the (opaque) hash of its contents
+  { path = "LICENSE", hash = 0xbd0eed23 },
+]
+
+
+[bans]
+# Lint level for when multiple versions of the same crate are detected
+multiple-versions = "warn"
+# Lint level for when a crate version requirement is `*`
+wildcards = "warn"
+workspace-default-features = "allow"
+external-default-features = "allow"
+allow = [
+  #{ name = "ansi_term", version = "=0.11.0" },
+]
+# List of crates to deny
+deny = [
+  # Each entry the name of a crate and a version range. If version is
+  # not specified, all versions will be matched.
+  #{ name = "ansi_term", version = "=0.11.0" },
+  #
+  # Wrapper crates can optionally be specified to allow the crate when it
+  # is a direct dependency of the otherwise banned crate
+  #{ name = "ansi_term", version = "=0.11.0", wrappers = [] },
+]

rustfmt.toml

@@ -0,0 +1,2 @@
+max_width = 90
+use_small_heuristics = "Max"

shell.nix

@@ -1,43 +1,42 @@
 let
-# Pinned nixpkgs, deterministic. Last updated to nixpkgs-unstable as of: 2024-07-23
-pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/68c9ed8bbed9dfce253cc91560bf9043297ef2fe.tar.gz") {};
+  # Pinned nixpkgs, deterministic. Last updated to nixpkgs-unstable as of: 2024-07-23
+  pkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/68c9ed8bbed9dfce253cc91560bf9043297ef2fe.tar.gz") {};
 
+  # Rolling updates, not deterministic.
+  # pkgs = import (fetchTarball("channel:nixpkgs-unstable")) {};
 
-# Rolling updates, not deterministic.
-# pkgs = import (fetchTarball("channel:nixpkgs-unstable")) {};
-
-alejandra =
-(import (builtins.fetchTarball {
-         url = "https://github.com/kamadorueda/alejandra/tarball/3.0.0";
-         sha256 = "sha256:18jm0d5xrxk38hw5sa470zgfz9xzdcyaskjhgjwhnmzd5fgacny4";
-         }) {})
-.outPath;
+  alejandra =
+    (import (builtins.fetchTarball {
+      url = "https://github.com/kamadorueda/alejandra/tarball/3.0.0";
+      sha256 = "sha256:18jm0d5xrxk38hw5sa470zgfz9xzdcyaskjhgjwhnmzd5fgacny4";
+    }) {})
+    .outPath;
 in
-pkgs.mkShell.override {
-  stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.clangStdenv;
-} {
-  packages = with pkgs; [
-    rustc
+  pkgs.mkShell.override {
+    stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.clangStdenv;
+  } {
+    packages = with pkgs; [
+      rustc
       cargo
       cargo-watch
-      cargo-nextest  # test runner
+      cargo-nextest # test runner
       cargo-deny
-      rust-analyzer  # rust lsp server
+      rust-analyzer # rust lsp server
       rustfmt
       clippy
-      taplo          # toml lsp server
+      taplo # toml lsp server
 
       awscli2
-      just           # script launcher with a make flavor
-      alejandra      # nix code formatter
-  ];
+      just # script launcher with a make flavor
+      alejandra # nix code formatter
+    ];
 
-  shellHook = ''
-    export PYTHONPATH=".:$PYTHONPATH"
+    shellHook = ''
+      export PYTHONPATH=".:$PYTHONPATH"
 
-    export AWS_ACCESS_KEY_ID=minio123
-    export AWS_SECRET_ACCESS_KEY=minio123
-    export AWS_DEFAULT_REGION=us-east-1
-    export RUSTFLAGS="-W unreachable-pub -W bare-trait-objects"
+      export AWS_ACCESS_KEY_ID=minio123
+      export AWS_SECRET_ACCESS_KEY=minio123
+      export AWS_DEFAULT_REGION=us-east-1
+      export RUSTFLAGS="-W unreachable-pub -W bare-trait-objects"
     '';
-}
+  }

src/dataset.rs

@@ -1,8 +1,9 @@
 use std::sync::Arc;
 
 use crate::{
-    AddNodeError, ArrayIndices, AttributesTable, ChunkPayload, Dataset, ManifestsTable, NodeData,
-    NodeStructure, Path, StructureTable, UpdateNodeError, UserAttributes, ZarrArrayMetadata,
+    AddNodeError, ArrayIndices, AttributesTable, ChunkPayload, Dataset, ManifestsTable,
+    NodeData, NodeStructure, Path, StructureTable, UpdateNodeError, UserAttributes,
+    ZarrArrayMetadata,
 };
 
 /// FIXME: what do we want to do with implicit groups?
@@ -46,10 +47,7 @@ impl Dataset {
     ) -> Result<(), UpdateNodeError> {
         match self.get_node(&path).await {
             None => Err(UpdateNodeError::NotFound),
-            Some(NodeStructure {
-                node_data: NodeData::Array(..),
-                ..
-            }) => {
+            Some(NodeStructure { node_data: NodeData::Array(..), .. }) => {
                 self.updated_arrays.insert(path, metadata);
                 Ok(())
             }
@@ -83,10 +81,7 @@ impl Dataset {
     ) -> Result<(), UpdateNodeError> {
         match self.get_node(&path).await {
             None => Err(UpdateNodeError::NotFound),
-            Some(NodeStructure {
-                node_data: NodeData::Array(..),
-                ..
-            }) => {
+            Some(NodeStructure { node_data: NodeData::Array(..), .. }) => {
                 self.set_chunks.insert((path, coord), data);
                 Ok(())
             }
@@ -98,11 +93,7 @@ impl Dataset {
 
     // FIXME: we should have errros here, not only None
     pub async fn get_node(&self, path: &Path) -> Option<NodeStructure> {
-        let structure = self
-            .storage
-            .fetch_structure(&self.structure_id)
-            .await
-            .ok()?;
+        let structure = self.storage.fetch_structure(&self.structure_id).await.ok()?;
         structure.get_node(path)
     }
 
@@ -121,11 +112,7 @@ impl Dataset {
     /// Files that are reused from previous commits are not returned because they don't need saving
     pub async fn consolidate(
         &mut self,
-    ) -> (
-        Arc<StructureTable>,
-        Vec<Arc<AttributesTable>>,
-        Vec<Arc<ManifestsTable>>,
-    ) {
+    ) -> (Arc<StructureTable>, Vec<Arc<AttributesTable>>, Vec<Arc<ManifestsTable>>) {
         todo!()
     }
 }

src/lib.rs

@@ -204,8 +204,7 @@ impl TryFrom<&[u8]> for ObjectId {
 
     fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
         let buf = value.try_into();
-        buf.map(ObjectId)
-            .map_err(|_| "Invalid ObjectId buffer length")
+        buf.map(ObjectId).map_err(|_| "Invalid ObjectId buffer length")
     }
 }
 
@@ -327,9 +326,18 @@ pub enum StorageError {
 /// Implementations are free to assume files are never overwritten.
 #[async_trait]
 trait Storage {
-    async fn fetch_structure(&self, id: &ObjectId) -> Result<Arc<StructureTable>, StorageError>; // FIXME: format flags
-    async fn fetch_attributes(&self, id: &ObjectId) -> Result<Arc<AttributesTable>, StorageError>; // FIXME: format flags
-    async fn fetch_manifests(&self, id: &ObjectId) -> Result<Arc<ManifestsTable>, StorageError>; // FIXME: format flags
+    async fn fetch_structure(
+        &self,
+        id: &ObjectId,
+    ) -> Result<Arc<StructureTable>, StorageError>; // FIXME: format flags
+    async fn fetch_attributes(
+        &self,
+        id: &ObjectId,
+    ) -> Result<Arc<AttributesTable>, StorageError>; // FIXME: format flags
+    async fn fetch_manifests(
+        &self,
+        id: &ObjectId,
+    ) -> Result<Arc<ManifestsTable>, StorageError>; // FIXME: format flags
 
     async fn write_structure(
         &self,