Merge branch 'master' into qa

.github/actions/generate-breadbox-client/action.yml

@@ -0,0 +1,27 @@
+# A re-usable github action which generates the breadbox client
+name: "generate-breadbox-client"
+description: "Auto-generate the breadbox client"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v2
+    - name: Install and configure Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.7.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+    - name: Install breadbox python dependencies
+      working-directory: ./breadbox
+      run: poetry install
+      shell: bash
+    - name: Export the breadbox API spec
+      working-directory: ./breadbox
+      run: |
+        poetry run ./bb export-api-spec ../breadbox-client/latest-breadbox-api.json
+      shell: bash
+    - name: Generate the breadbox client
+      run: |
+        pip install openapi-python-client==0.21.1
+        openapi-python-client generate --meta=none --output-path breadbox-client/breadbox_client --path breadbox-client/latest-breadbox-api.json --overwrite
+      shell: bash

.github/workflows/build_breadbox.yml

@@ -64,15 +64,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          version: 1.7.1
-          virtualenvs-create: true
-          virtualenvs-in-project: true
+      - name: Generate breadbox client
+        uses: ./.github/actions/generate-breadbox-client # defined as a re-usable action
       - name: check that the breadbox client passes pyright
+        working-directory: ./breadbox-client
         run: |
-          cd breadbox-client && poetry install && poetry run pyright breadbox_facade
+          poetry install && poetry run pyright breadbox_facade
   build-docker:
     runs-on: ubuntu-latest
     steps:
@@ -93,9 +90,6 @@ jobs:
           yarn --cwd frontend "build:elara"
       - name: build docker image
         run: bash breadbox/build-docker-image.sh ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}
-      - name: check that the breadbox client is in sync with the latest API spec
-        run: |
-          docker run -v $PWD/breadbox-client/latest-breadbox-api.json:/latest-breadbox-api.json ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} poetry run ./bb check-api /latest-breadbox-api.json
       - name: Run pyright-ratchet
         run: |
           docker run -v $PWD/breadbox/pyright-ratchet-errors.txt:/install/breadbox/pyright-ratchet-errors.txt ${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }} poetry run pyright-ratchet
@@ -186,13 +180,10 @@ jobs:
       - name: Get bumped version
         run: echo "${{ steps.current-version.outputs.bumped_version}}"
 
-  publish-breadbox-and-client:
+  publish-breadbox-client:
     needs: bump-versions
     if: ${{ (needs.bump-versions.outputs.current_version != needs.bump-versions.outputs.bumped_version) && (github.ref_name == 'master') }}
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./breadbox
     steps:
       - name: Check out
         uses: actions/checkout@v3
@@ -218,17 +209,8 @@ jobs:
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Install dependencies
-        run: poetry install
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-      - name: Set up for publishing breadbox
-        working-directory: ./breadbox
-        run: |
-          poetry self add keyrings.google-artifactregistry-auth
-          poetry config repositories.public-python https://us-central1-python.pkg.dev/cds-artifacts/public-python/
-      - name: Publish new breadbox version to Artifact Registry
-        working-directory: ./breadbox
-        run: poetry publish --build --repository public-python
+      - name: Generate breadbox client
+        uses: ./.github/actions/generate-breadbox-client # defined as a re-usable action
       - name: Set up for publishing breadbox client
         working-directory: ./breadbox-client
         run: |

.gitignore

@@ -174,13 +174,16 @@ app_datasets/
 breadbox/install
 breadbox/breadbox/static
 breadbox/breadbox/result
-# generated file
+
+# generated files
 portal-backend/depmap/utilities/color_palette.py
 breadbox/app_datasets
+breadbox_client/
+latest-breadbox-api.json
 secrets
 
 # data access logs
 portal-backend/access.log
 
 # predictability output_data from local run
-output_data/
+output_data/

breadbox-client-generator/README.md

@@ -0,0 +1,3 @@
+# Breadbox Client Generator
+
+A poetry project used for the generation of the breadbox client. This is defined as a separate project because the `openapi-python-client` has dependency conflicts with breadbox.

breadbox-client-generator/pyproject.toml

@@ -13,4 +13,4 @@ openapi-python-client = "^0.21.1"
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"

breadbox-client/README.md

@@ -1,15 +1,43 @@
-# The Breadbox Facade
+# The Breadbox Client
+
+A client library for accessing Breadbox
+
+Includes both an auto-generated `breadbox_client` module and a human-curated `breadbox_facade` which defines a more user-friendly interface for the client. 
+
+
+## Auto-Generating the breadbox_client
+When changes are merged to master, the `breadbox-client/breadbox_client` directory will be auto-generated before the breadbox-client module is published.
+
+In general, the `breadbox_client` directory is auto-generated in a github action. It is not checked in to the repo.
+When changes are merged to master, the `breadbox-client/breadbox_client` directory will be auto-generated before the breadbox-client module is published.
+
+To generate the client for local development:
+1. Make sure you have the `breadbox-client-generator` poetry project installed: 
+    ```
+    cd ../breadbox-client-generator
+    poetry install
+    ```
+2. Generate the breadbox client:
+    ```
+    cd ../breadbox 
+    poetry run ./bb update-client
+    ```
 
-A user-friendly facade for the autogenerated breadbox client.
 
-### This includes improvements such as:
+## The breadbox Facade 
+
+#### Includes improvements such as:
 
 - **Simpler return types**. For example, the facade's get_datasets method returns `list[DatasetResponse]` instead of `Optional[Union[HTTPError, HTTPValidationError, List["DatasetResponse"]]]`
 - **Better error handling**: Exceptions are raised instead of being returned
 - **Constructors for custom types** For example, this facade accepts DataFrames instead of taking `breadbox_client.types.File` objects.
 - **Better handling of None values** in cases where the autogenerated client wants to be given UNSET instead of None
 
-### To publish a new version:
+## To publish a new version of this module:
+
+**Option 1 (preferred)**: Merge your changes to the main branch. This module will be automatically updated and published. 
+
+**Option 2**: Publish from local changes
 
 To setup for publishing (Based on https://medium.com/google-cloud/python-packages-via-gcps-artifact-registry-ce1714f8e7c1 )
 
@@ -62,10 +90,6 @@ poetry source add --priority=supplemental public-python https://us-central1-pyth
 poetry add --source public-python breadbox-facade
 ```
 
-# breadbox-client
-
-A client library for accessing Breadbox
-
 ## Usage
 
 First, create a client: