add ski prepull, fix basic tests

ucsd-ets · Dec 29, 2024 · d85116a · d85116a
1 parent 23876e2
commit d85116a
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 11 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -3,6 +3,7 @@ name: AUTO Build and Test
 env:
   DOCKER_CLIENT_TIMEOUT: "300"
   COMPOSE_HTTP_TIMEOUT: "300"
+  SKIP_PREPULL: "false"
 
 on:
   push:
@@ -23,6 +24,11 @@ on:
       - ".github/workflows/main.yml"
 
   workflow_dispatch:
+    inputs:
+      prepull_images:
+        description: "Prepull images prior to build"
+        required: false
+        default: "false"
 
 jobs:
   start-runner:
@@ -57,9 +63,16 @@ jobs:
       !contains(github.event.pull_request.title, 'skip ci')
     steps:
       - name: Checkout after Free Space
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Setup ENV vars
+        run: |
+          if [[ "${{ github.event.inputs.prepull_images || 'false' }}" == "true" ]]; then
+            echo "SKIP_PREPULL=true" >> $GITHUB_ENV
+          else
+            echo "SKIP_PREPULL=false" >> $GITHUB_ENV
 
       - name: Docker/ENV cleanup Cleanup
         run: |
@@ -77,10 +90,10 @@ jobs:
           sudo dnf remove python3-requests -y
           python3 -m pip install -r scripts/requirements.txt
       
-      - name: Install selenium dependecies for r studio
-        run: |
-          chmod +x ./scripts/selenium_setup.sh 
-          ./scripts/selenium_setup.sh
+      #- name: Install selenium dependecies for r studio
+      #  run: |
+      #    chmod +x ./scripts/selenium_setup.sh 
+      #    ./scripts/selenium_setup.sh
 
       - name: Install pydoit fixed
         run: |
@@ -106,7 +119,7 @@ jobs:
           mkdir -p logs
 
       - name: Clone Wiki
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{github.repository}}.wiki
           path: wiki
@@ -142,7 +155,7 @@ jobs:
 
       - name: Archive artifacts and logs
         if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build-artifacts
           path: |

diff --git a/images/tests_common/test_packages.py b/images/tests_common/test_packages.py
@@ -98,7 +98,11 @@
     "jupyterlab-git",
     "jupyter-pluto-proxy",
     "python-lsp-server",
-     "mamba[version='<2.0.0']",
+    "mamba[version='<2.0.0']",
+    "mamba",
+    "jupyterhub-singleuser",
+    "conda-forge::blas=[build",
+    "nose"
 ]
 
 

diff --git a/scripts/runner.py b/scripts/runner.py
@@ -201,10 +201,17 @@ def build_and_test_containers(
     # prepull images inorder to use cache
     # AND mark rebuild for those without cache yet
     last_t = datetime.datetime.now()  # to log timestamp
+    skip_prepull = os.getenv('SKIP_PREPULL', 'false')
+    skip_prepull = skip_prepull.lower() == 'true'
+
     for i, node in enumerate(node_order):
-        if not docker_adapter.pull_build_cache(node):
-            # self doesn't exist on Dockerhub, rebuild
-            logger.info(f"{node.full_image_name} doesn't exist on Dockerhub. \
+        if skip_prepull:
+            node.prepull = False
+            node_order[i].rebuild = True
+            logger.info(f"Skipping prepull for {node.full_image_name}.")
+        if not skip_prepull and not docker_adapter.pull_build_cache(node):
+            # self doesn't exist on GHCR, rebuild
+            logger.info(f"{node.full_image_name} doesn't exist on GHCR. \
                 It will be built from stable image's cache and save future build time.")
             node_order[i].rebuild = True
     last_t, m, s = get_time_duration(last_t)