Alignment Subrepo

ZettaAI · Nov 30, 2023 · 9ddf881 · 9ddf881
1 parent d604cf6
commit 9ddf881
Show file tree

Hide file tree

Showing 16 changed files with 95 additions and 1,236 deletions.
diff --git a/.github/workflows/docs_deployment.yaml b/.github/workflows/docs_deployment.yaml
@@ -19,12 +19,22 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
         with:
           lfs: 'false'
-          submodules: 'recursive'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
       - name: Setup Python
         uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml
@@ -21,12 +21,23 @@ jobs:
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
         with:
           lfs: 'false'
-          submodules: 'recursive'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
+
       - name: Get changed files
         uses: dorny/paths-filter@v2
         id: filter
@@ -75,12 +86,22 @@ jobs:
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
         with:
           lfs: 'false'
-          submodules: 'recursive'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
       - name: Get changed files
         uses: dorny/paths-filter@v2
         id: filter
@@ -116,12 +137,22 @@ jobs:
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
         with:
           lfs: 'false'
-          submodules: 'recursive'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
       - name: Get changed files
         uses: dorny/paths-filter@v2
         id: filter
@@ -155,12 +186,22 @@ jobs:
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
         with:
           lfs: 'false'
-          submodules: 'recursive'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
       - name: Get changed files
         uses: dorny/paths-filter@v2
         id: filter

diff --git a/.github/workflows/testing_integration.yaml b/.github/workflows/testing_integration.yaml
@@ -17,12 +17,22 @@ jobs:
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Get token from Github App
+        uses: actions/create-github-app-token@v1
+        id: app_token
         with:
-          lfs: 'true'
-          submodules: 'recursive'
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PEM }}
+          # owner is required, otherwise the creds will fail the checkout step
+          owner: ${{ github.repository_owner }}
+
+      - name: Checkout from GitHub
+        uses: actions/checkout@v4
+        with:
+          lfs: 'false'
+          submodules: true
           ssh-key: ${{ secrets.git_ssh_key  }}
+          token: ${{ steps.app_token.outputs.token }}
       - name: Get changed files
         uses: dorny/paths-filter@v2
         id: filter

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "zetta_utils/alignment"]
+	path = zetta_utils/alignment
+	url = [email protected]:ZettaAI/alignment.git
diff --git a/zetta_utils/__init__.py b/zetta_utils/__init__.py
@@ -31,9 +31,15 @@ def try_load_train_inference():  # pragma: no cover
         ...
 
 
+def try_load_submodules():  # pragma: no cover
+    try:
+        from . import alignment
+    except ImportError:
+        ...
+
+
 def load_inference_modules():
     from . import (
-        alignment,
         augmentations,
         convnet,
         mazepa,
@@ -47,10 +53,11 @@ def load_inference_modules():
     from .layer.volumetric import cloudvol
     from .message_queues import sqs
 
+    try_load_submodules()
+
 
 def load_training_modules():
     from . import (
-        alignment,
         augmentations,
         convnet,
         mazepa,
@@ -62,5 +69,7 @@ def load_training_modules():
     from .layer import volumetric
     from .layer.volumetric import cloudvol
 
+    try_load_submodules()
+
 
 try_load_train_inference()
diff --git a/zetta_utils/alignment b/zetta_utils/alignment
diff --git a/zetta_utils/alignment/__init__.py b/zetta_utils/alignment/__init__.py