getting started, feels very doable

[x] proof of concept, pure rust and only pulling images for now
[x] using kanal for message passing
[x] async page pull
[x] small cli with arguments
[x] reuse the same client over time, and the same connection pool across the threads
[x] image crop and resize
[x] pull all the payloads
[ ] nice threadpool handling, workstealing and all ideally
[ ] image encoding
[ ] bulletproof all paths
[ ] unit tests
[x] CI
[ ] CD
[ ] python wrap

.github/workflows/rust.yml

@@ -0,0 +1,49 @@
+name: Rust CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        rust: [stable]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Check formatting
+        run: cd rust/datago && cargo fmt --all -- --check
+
+      - name: Build
+        run: cd rust/datago && cargo build --verbose
+
+      - name: Run tests
+        run: cd rust/datago && cargo test --verbose

.gitignore

@@ -31,3 +31,5 @@ build
 
 __pycache__
 *.pyc
+
+target/debug

rust/datago/.gitignore

@@ -0,0 +1 @@
+target