improvement, seed: reduce size of seed containers and speed up python…

… and java tests (#3011) * update python * update java and ts too * eslint Dockerfiles * ignore literal for fastapi * update .fernignore * publish containers * try again * try publishing again * retry * try agin * clone repo * fix * update python and java image dep * run yarn build after yarn install * Update build-seed.yml * Update Dockerfile.python * try installing xz * Update Dockerfile.python * Update build-seed.yml * Update testWorkspaceFixtures.ts * update java model too and see if reducing parallelism helps get it to local * run when this file is updated * Update seed.yml * Update seed.yml * Update seed.yml * updates should work * more files * republish python-seed * update allowed failures and get seed to green * fix ruby sdk * fix java sdk --------- Co-authored-by: dsinghvi <[email protected]>
fern-api · Feb 21, 2024 · 953a306 · 953a306
1 parent 7ef4466
commit 953a306
Show file tree

Hide file tree

Showing 129 changed files with 8,762 additions and 698 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -10,4 +10,4 @@ packages/ir-sdk/**
 packages/seed/**
 generators/typescript/utils/core-utilities/**
 seed/**
-
+docker/seed/**
diff --git a/.github/workflows/build-seed.yml b/.github/workflows/build-seed.yml
@@ -0,0 +1,74 @@
+name: Build seed containers
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+# Cancel previous workflows on previous push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      packages: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            python: 
+              - 'docker/seed/Dockerfile.python'
+              - 'generators/python/poetry.lock'
+              - '.github/workflows/build-seed.yml'
+            ts: 
+              - 'docker/seed/Dockerfile.ts'
+              - '.github/workflows/build-seed.yml'
+            java: 
+              - 'docker/seed/Dockerfile.java'
+              - '.github/workflows/build-seed.yml'
+  build-container:
+    strategy:
+      matrix:
+        # Parse JSON array containing names of all filters matching any of changed files
+        # e.g. ['package1', 'package2'] if both package folders contains changes
+        package: ${{ fromJSON(needs.changes.outputs.packages) }}
+    runs-on: ubuntu-latest
+    needs: changes
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: fernapi
+          password: ${{ secrets.FERN_API_DOCKERHUB_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: fernapi/${{ matrix.package }}-seed
+          tags: |
+            type=sha
+            latest
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/seed/Dockerfile.${{ matrix.package }}
+          push: true
+          # use short SHA if modded, or just use latest if nothing changed
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/seed.yml b/.github/workflows/seed.yml
@@ -32,6 +32,7 @@ jobs:
         with:
           filters: |
             seed: 
+              - '.github/workflows/seed.yml'
               - 'packages/seed/**'
               - 'test-definitions/**'
             ruby: 'generators/ruby/**'
@@ -56,15 +57,15 @@ jobs:
 
       - name: Install Seed
         env:
-          FORCE_COLOR: "2"      
+          FORCE_COLOR: "2"
         run: npm install -g @fern-api/seed-cli
 
       - name: Yarn Install
         run: yarn install
 
       - name: Seed Test
         run: |
-          yarn seed:local test --workspace ruby-model --parallel 16
+          yarn seed:local test --workspace ruby-model
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -82,17 +83,17 @@ jobs:
           node-version: 18
           cache: "yarn"
 
-      - name: Install Seed      
+      - name: Install Seed
         run: npm install -g @fern-api/seed-cli
 
       - name: Yarn Install
         run: yarn install
 
       - name: Seed Test
         env:
-          FORCE_COLOR: "2"         
+          FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace ruby-sdk --parallel 16
+          yarn seed:local test --workspace ruby-sdk
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -122,9 +123,9 @@ jobs:
 
       - name: Seed Test
         env:
-          FORCE_COLOR: "2"       
+          FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace pydantic --parallel 16
+          yarn seed:local test --workspace pydantic
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -154,9 +155,9 @@ jobs:
 
       - name: Seed Test
         env:
-          FORCE_COLOR: "2"           
+          FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace python-sdk --parallel 16
+          yarn seed:local test --workspace python-sdk --parallel 10
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -186,9 +187,9 @@ jobs:
 
       - name: Seed Test
         env:
-          FORCE_COLOR: "2"      
+          FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace fastapi --parallel 16
+          yarn seed:local test --workspace fastapi
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -213,7 +214,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace openapi --parallel 16
+          yarn seed:local test --workspace openapi
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -238,7 +239,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace postman --parallel 16
+          yarn seed:local test --workspace postman
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -263,7 +264,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace java-sdk --parallel 16
+          yarn seed:local test --workspace java-sdk --parallel 10
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -288,7 +289,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace java-model --parallel 16
+          yarn seed:local test --workspace java-model --parallel 10
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -313,7 +314,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace java-spring --parallel 16
+          yarn seed:local test --workspace java-spring
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -336,9 +337,9 @@ jobs:
 
       - name: Seed Test
         env:
-          FORCE_COLOR: "2"      
+          FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace ts-sdk --parallel 16
+          yarn seed:local test --workspace ts-sdk
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
@@ -363,7 +364,7 @@ jobs:
         env:
           FORCE_COLOR: "2"
         run: |
-          yarn seed:local test --workspace ts-express --parallel 16
+          yarn seed:local test --workspace ts-express  --parallel 1
 
       - name: Ensure no changes to git-tracked files
         run: git --no-pager diff --exit-code
diff --git a/docker/seed/Dockerfile.java b/docker/seed/Dockerfile.java
@@ -0,0 +1,57 @@
+# Small ubuntu base image
+FROM redhat/ubi9:latest
+
+# Install sdkman
+RUN yum update -y
+RUN yum -y install git zip unzip
+RUN git clone https://github.com/jenv/jenv.git ~/.jenv
+RUN echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bash_profile
+RUN echo 'eval "$(jenv init -)"' >> ~/.bash_profile
+RUN source ~/.bash_profile
+
+RUN set -eux; \
+    ARCH="$(uname -m)"; \
+    case "${ARCH}" in \
+       aarch64|arm64) \
+         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u292-b10/OpenJDK8U-jdk_aarch64_linux_hotspot_8u292b10.tar.gz'; \
+         ;; \
+       amd64|x86_64) \
+         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u292-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u292b10.tar.gz'; \
+         ;; \
+       *) \
+         echo "Unsupported arch: ${ARCH}"; \
+         exit 1; \
+         ;; \
+    esac; \
+    curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \
+    mkdir -p /opt/java/openjdk8; \
+    cd /opt/java/openjdk8; \
+    tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
+    rm -rf /tmp/openjdk.tar.gz;
+
+RUN source ~/.bash_profile && jenv add /opt/java/openjdk8
+
+RUN set -eux; \
+    ARCH="$(uname -m)"; \
+    case "${ARCH}" in \
+       aarch64|arm64) \
+         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.9.1%2B1/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.9.1_1.tar.gz'; \
+         ;; \
+       amd64|x86_64) \
+         BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.9.1%2B1/OpenJDK11U-jdk_x64_linux_hotspot_11.0.9.1_1.tar.gz'; \
+         ;; \
+       *) \
+         echo "Unsupported arch: ${ARCH}"; \
+         exit 1; \
+         ;; \
+    esac; \
+    curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \
+    mkdir -p /opt/java/openjdk11; \
+    cd /opt/java/openjdk11; \
+    tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
+    rm -rf /tmp/openjdk.tar.gz;
+
+RUN source ~/.bash_profile && jenv add /opt/java/openjdk11
+RUN source ~/.bash_profile && jenv global 11
+RUN source ~/.bash_profile && jenv enable-plugin export
+RUN source ~/.bash_profile && jenv enable-plugin gradle
diff --git a/docker/seed/Dockerfile.python b/docker/seed/Dockerfile.python
@@ -0,0 +1,32 @@
+# Small ubuntu base image
+FROM redhat/ubi9:latest
+
+# Setup the environment, can run pytest, poetry install, etc.
+COPY seed/python-sdk/imdb/pyproject.toml ./pyproject.toml
+COPY seed/python-sdk/imdb/poetry.lock ./poetry.lock
+COPY seed/python-sdk/imdb/src ./src
+
+# Setup pyenv
+RUN yum update -y
+RUN yum -y install git gcc zlib-devel openssl-devel xz-devel
+RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv
+ENV PYENV_ROOT="${HOME}/.pyenv"
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"
+RUN eval "$(pyenv init --path)"
+
+RUN pyenv install 3.9 || true
+RUN pyenv install 3.8 || true
+
+# Setup poetry
+RUN pyenv global 3.9
+RUN pip install poetry  &&\
+  poetry config virtualenvs.create false &&\
+  poetry install
+RUN pip install mypy==1.8.0
+
+# Setup poetry
+RUN pyenv global 3.8
+RUN pip install poetry  &&\
+  poetry config virtualenvs.create false &&\
+  poetry install
+RUN pip install mypy==1.8.0
diff --git a/docker/seed/Dockerfile.ts b/docker/seed/Dockerfile.ts
@@ -0,0 +1,4 @@
+FROM node:lts-slim
+
+# Installs tsc
+RUN npm install -g typescript
diff --git a/packages/seed/src/cli.ts b/packages/seed/src/cli.ts
@@ -40,7 +40,7 @@ function addTestCommand(cli: Argv) {
                 .option("custom-fixture", {
                     type: "string",
                     demandOption: false,
-                    description: "Path to the api directory"
+                    description: "Path to the API directory"
                 })
                 .option("fixture", {
                     type: "array",

diff --git a/seed/csharp-model/alias/ir.json b/seed/csharp-model/alias/ir.json
diff --git a/seed/csharp-model/api-wide-base-path/ir.json b/seed/csharp-model/api-wide-base-path/ir.json