Merge Upstream

.github/workflows/build-api.yml

@@ -0,0 +1,113 @@
+name: Build das api components
+# This workflow uses github runners.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# This may be adjusted to whatever suits best your runners config.
+# Current config will build on manual trigger or pull-request (each push)
+on:
+  # pull_request can be removed, to save minutes on github runners
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+
+env:
+   CARGO_TERM_COLOR: always
+
+jobs:
+  build-api:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, ubuntu-22.04]
+    # This can be also be runned on self-hosted github runners
+    runs-on: ["${{ matrix.os }}"]
+
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v3
+      # This step can be omited, to save storage space on the organization account
+      # Build process will take longer
+      - name: set build cache
+        uses: actions/cache@v3
+        with:
+            path: |
+                ~/.cargo/bin/
+                ~/.cargo/registry/index/
+                ~/.cargo/registry/cache/
+                ~/.cargo/git/db/
+                digital-asset-rpc-infrastructure/target/
+            key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }}
+            restore-keys: |
+              ${{ matrix.os }}_digital-asset-rpc-infrastructure
+
+      - name: build digital asset rpc infra
+        run: cargo build --verbose --release
+
+      - name: rename binaries for ubuntu22 release
+        if: matrix.os == 'ubuntu-22.04'
+        run: |
+          mv target/release/nft_ingester target/release/nft_ingester22
+          mv target/release/fetch_trees target/release/fetch_trees22 
+          mv target/release/migration target/release/migration22
+          mv target/release/das_api target/release/das_api22
+
+      # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them 
+      # Omiting this will save on storage consumption on the account
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-22.04'
+        uses: actions/[email protected]
+        with:
+          name: nft_ingester22
+          path: target/release/nft_ingester22
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-22.04'
+        uses: actions/[email protected]
+        with:
+          name: das_api22
+          path: target/release/das_api22
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-22.04'
+        uses: actions/[email protected]
+        with:
+          name: migration22
+          path: target/release/migration22
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-22.04'
+        uses: actions/[email protected]
+        with:
+          name: fetch-trees22
+          path: target/release/fetch_trees22
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-20.04'
+        uses: actions/[email protected]
+        with:
+          name: nft_ingester
+          path: target/release/nft_ingester
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-20.04'
+        uses: actions/[email protected]
+        with:
+          name: das_api
+          path: target/release/das_api
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-20.04'
+        uses: actions/[email protected]
+        with:
+          name: migration
+          path: target/release/migration
+
+      - name: Publish artifact
+        if: matrix.os == 'ubuntu-20.04'
+        uses: actions/[email protected]
+        with:
+          name: fetch-trees
+          path: target/release/fetch_trees

.github/workflows/test.yml

@@ -1,35 +1,27 @@
-name: Build das api components
-# This workflow uses github runners.
+name: Check lock file, fmt, clippy
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-# This may be adjusted to whatever suits best your runners config.
-# Current config will build on manual trigger or pull-request (each push)
 on:
-  # pull_request can be removed, to save minutes on github runners
   pull_request:
-  workflow_dispatch:
   push:
     branches:
       - 'main'
+  workflow_dispatch:
 
 env:
    CARGO_TERM_COLOR: always
 
 jobs:
-  build-api:
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
-    # This can be also be runned on self-hosted github runners
-    runs-on: ["${{ matrix.os }}"]
+  test:
+    runs-on: ubuntu-22.04
 
     steps:
       - name: checkout repo
         uses: actions/checkout@v3
-      # This step can be omited, to save storage space on the organization account
-      # Build process will take longer
+
       - name: set build cache
         uses: actions/cache@v3
         with:
@@ -39,83 +31,19 @@ jobs:
                 ~/.cargo/registry/cache/
                 ~/.cargo/git/db/
                 digital-asset-rpc-infrastructure/target/
-            key: ${{ matrix.os }}_digital-asset-rpc-infrastructure_${{ hashFiles('digital-asset-rpc-infrastructure/Cargo.lock') }}
-            restore-keys: |
-              ${{ matrix.os }}_digital-asset-rpc-infrastructure
-
-      - name: build digital asset rpc infra
-        run: cargo build --verbose --release
-
-      - name: build das_api
-        working-directory: das_api
-        run: cargo build --verbose --release
+            key: cargo-${{ hashFiles('**/Cargo.lock') }}-0001
 
-      - name: build migration
-        working-directory: migration
-        run: cargo build --verbose --release
-
-      - name: rename binaries for ubuntu22 release
-        if: matrix.os == 'ubuntu-22.04'
+      # Cargo.lock
+      - name: Check lock file
         run: |
-          mv target/release/nft_ingester target/release/nft_ingester22
-          mv target/release/fetch_trees target/release/fetch_trees22 
-          mv das_api/target/release/das_api das_api/target/release/das_api22 
-          mv migration/target/release/migration migration/target/release/migration22
+          cargo tree
+          git checkout Cargo.lock
+          cargo tree --frozen
 
-      # This steps can be omited to save space, are mostly in place to validate binaries (manually) and path to them 
-      # Omiting this will save on storage consumption on the account
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-22.04'
-        uses: actions/[email protected]
-        with:
-          name: nft_ingester22
-          path: target/release/nft_ingester22
+      # fmt
+      - name: Check fmt
+        run: cargo fmt --all -- --check
 
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-22.04'
-        uses: actions/[email protected]
-        with:
-          name: das_api22
-          path: das_api/target/release/das_api22
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-22.04'
-        uses: actions/[email protected]
-        with:
-          name: migration22
-          path: migration/target/release/migration22
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-22.04'
-        uses: actions/[email protected]
-        with:
-          name: fetch-trees22
-          path: migration/target/release/fetch_trees22
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-20.04'
-        uses: actions/[email protected]
-        with:
-          name: nft_ingester
-          path: target/release/nft_ingester
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-20.04'
-        uses: actions/[email protected]
-        with:
-          name: das_api
-          path: das_api/target/release/das_api
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-20.04'
-        uses: actions/[email protected]
-        with:
-          name: migration
-          path: migration/target/release/migration
-
-      - name: Publish artifact
-        if: matrix.os == 'ubuntu-20.04'
-        uses: actions/[email protected]
-        with:
-          name: fetch-trees
-          path: target/release/fetch_trees
+      # clippy
+      - name: Check clippy
+        run: cargo clippy --all-targets --tests

.gitignore

@@ -23,10 +23,3 @@ programs
 *.iml
 skaffold-state.json
 test-programs
-
-# Rust build dirs
-/das_api/target
-/migration/target
-/tests/*/target
-/*/target
-/target

Api.Dockerfile

@@ -1,21 +1,41 @@
 FROM rust:1.73-bullseye AS chef
 RUN cargo install cargo-chef
 FROM chef AS planner
-COPY das_api /rust/das_api/
+
+RUN mkdir /rust
+COPY Cargo.toml /rust
+COPY das_api /rust/das_api
+COPY digital_asset_types /rust/digital_asset_types
+COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
+COPY migration /rust/migration
+COPY nft_ingester /rust/nft_ingester
+COPY tools /rust/tools
+
 WORKDIR /rust/das_api
-RUN cargo chef prepare --recipe-path recipe.json
+RUN cargo chef prepare --recipe-path /rust/das_api/recipe.json
 
 FROM chef AS builder
 RUN apt-get update -y && \
     apt-get install -y build-essential make git
+
+RUN mkdir /rust
+COPY Cargo.toml /rust
+COPY das_api /rust/das_api
 COPY digital_asset_types /rust/digital_asset_types
-WORKDIR /
-RUN mkdir -p /rust/das_api
+COPY metaplex-rpc-proxy /rust/metaplex-rpc-proxy
+COPY migration /rust/migration
+COPY nft_ingester /rust/nft_ingester
+COPY tools /rust/tools
+
 WORKDIR /rust/das_api
 COPY --from=planner /rust/das_api/recipe.json recipe.json
+
 # Build dependencies - this is the caching Docker layer!
-COPY das_api/Cargo.toml .
-COPY das_api .
+# TODO: Fix this.  For now we are building without the cached dependencies as there's apparently
+# some problem with digital-asset-types feature activation.
+
+# RUN cargo chef cook --release --recipe-path recipe.json --target-dir /rust/target --all-features
+
 # Build application
 RUN cargo build --release
 
@@ -29,7 +49,7 @@ ENV TZ=Etc/UTC \
 RUN groupadd $APP_USER \
     && useradd -g $APP_USER $APP_USER \
     && mkdir -p ${APP}
-COPY --from=builder /rust/das_api/target/release/das_api ${APP}
+COPY --from=builder /rust/target/release/das_api ${APP}
 RUN chown -R $APP_USER:$APP_USER ${APP}
 USER $APP_USER
 WORKDIR ${APP}