diff --git a/.github/.devcontainer/Dockerfile b/.github/.devcontainer/Dockerfile new file mode 100644 index 000000000..34e876d43 --- /dev/null +++ b/.github/.devcontainer/Dockerfile @@ -0,0 +1,190 @@ +FROM ubuntu:noble-20241118.1 + +# Set safer bash scripts. +SHELL ["/bin/bash", "-euxo", "pipefail", "-c"] + +# List of build arguments. +# https://github.com/nektos/act +ARG actVersion="0.2.71" +# https://github.com/sharkdp/hyperfine +ARG hyperfineVersion="1.19.0" +# https://www.npmjs.com/package/@devcontainers/cli +ARG devcontainerCliVersion="0.72.0" +# https://pypi.org/project/poetry +ARG poetryVersion="1.8.5" +# https://github.com/astral-sh/uv +ARG uvVersion="0.5.14" +# https://docs.posit.co/resources/install-r/#specify-r-version +ARG rVersion="4.4.1" +# https://aquasecurity.github.io/trivy +ARG trivyVersion="0.58.1" +# https://github.com/rstudio/renv +ARG renvVersion="1.0.11" +# https://nodejs.org/en/about/previous-releases +ARG nodeVersionMajor="22" +# https://pypi.org/project/pipenv/ +ARG pipenvVersion="2024.4.0" +# https://github.com/pnpm/pnpm/releases +ARG pnpmVersion="9.15.2" +# List of Python versions separated by spaces +ARG pyenvPythonVersions="3.13.1" +# https://github.com/SonarSource/sonar-scanner-cli/releases +ARG sonarScannerVersion="6.2.1.4610" +# https://github.com/hadolint/hadolint +ARG hadolintVersion="2.12.0" +# The version of this dev container image. +ARG devcontainerVersion="" +# The username of the non-root user. +# The user `ubuntu` is available since Ubuntu 24.04. +ENV user="ubuntu" + +# Set environment variables. +ENV DEVCONTAINER_VERSION=${devcontainerVersion} +ENV LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 + +# Install system packages and other tools. +# hadolint ignore=DL3008 +RUN apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install --no-install-recommends -qq -y \ + ca-certificates curl git bash-completion gnupg2 lsb-release ssh sudo \ + python3-pip python3-dev python-is-python3 pipx openjdk-17-jdk \ + htop unzip vim wget lsof iproute2 build-essential \ + kafkacat jq ca-certificates-java gdebi-core \ + # Required by AWS CLI. + mandoc \ + # Required for setting up locales. + locales \ + # Required by pyenv. + make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ + xz-utils tk-dev libffi-dev liblzma-dev \ + # Required by Hadolint. + shellcheck \ + # Add Node.js repository. + && curl -fsSL https://deb.nodesource.com/setup_${nodeVersionMajor}.x -o nodesource_setup.sh \ + && bash nodesource_setup.sh \ + # Add GitHub CLI repository. + && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \ + gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | \ + tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + # Add ngrok repository. + && curl -fsSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \ + && echo "deb https://ngrok-agent.s3.amazonaws.com bullseye main" | tee /etc/apt/sources.list.d/ngrok.list \ + # Add hashicorp repository. + && curl -fsSL https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list \ + # Install additional packages. + && apt-get update -qq -y \ + && apt-get install --no-install-recommends -qq -y nodejs gh ngrok terraform vault \ + # Enable corepack. + && corepack enable \ + # Fix Vault CLI. + # See https://github.com/hashicorp/vault/issues/10924 + && setcap -r /usr/bin/vault \ + # Set up UTF-8 locale. + && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen \ + # Install R (must be done before clearing the apt cache) + # See https://github.com/rstudio/r-builds + && curl -fsSL "https://cdn.rstudio.com/r/ubuntu-2404/pkgs/r-${rVersion}_1_amd64.deb" -o /tmp/r_amd64.deb \ + && gdebi --non-interactive /tmp/r_amd64.deb \ + && rm -fr /tmp/r_amd64.deb \ + && ln -s /opt/R/${rVersion}/bin/R /usr/local/bin/R \ + && ln -s /opt/R/${rVersion}/bin/Rscript /usr/local/bin/Rscript \ + && R -e "options(repos = c(POSIT = \"https://packagemanager.posit.co/all/__linux__/jammy/latest\", CRAN = \"https://mirror.las.iastate.edu/CRAN\")); install.packages(\"renv\", version = \"${renvVersion}\")" \ + # Cleanup. + && apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* + +# Install Trivy. +RUN curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${trivyVersion}/trivy_${trivyVersion}_Linux-64bit.deb" -o /tmp/trivy.deb \ + && dpkg -i /tmp/trivy.deb \ + && rm -fr /tmp/trivy.deb + +# Install act. +RUN curl -fsSL "https://raw.githubusercontent.com/nektos/act/v${actVersion}/install.sh" | bash - + +# Install AWS CLI. +RUN curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip \ + && unzip awscliv2.zip \ + && ./aws/install \ + && rm -fr awscliv2.zip ./aws \ + # Install AWS Session Manager plugin. + && curl -fsSL "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o /tmp/session-manager-plugin.deb \ + && dpkg -i /tmp/session-manager-plugin.deb \ + && rm -fr /tmp/session-manager-plugin.deb + +# Install AWS SAM CLI. +RUN curl -Lo aws-sam-cli-linux-x86_64.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip \ + && unzip aws-sam-cli-linux-x86_64.zip -d sam-installation \ + && ./sam-installation/install \ + && rm -rf aws-sam-cli-linux-x86_64.zip sam-installation \ + && sam --version + +# Install the devcontainer CLI. +RUN npm install -g "@devcontainers/cli@${devcontainerCliVersion}" + +# Install Hadolint. +RUN curl -fsSL https://github.com/hadolint/hadolint/releases/download/v${hadolintVersion}/hadolint-Linux-x86_64 -o hadolint \ + && mv hadolint /usr/local/bin/. \ + && chmod +x /usr/local/bin/hadolint + +# Install hyperfine. +RUN curl -fsSL "https://github.com/sharkdp/hyperfine/releases/download/v${hyperfineVersion}/hyperfine_${hyperfineVersion}_amd64.deb" \ + -o /tmp/hyperfine.deb \ + && apt-get install --no-install-recommends -qq -y /tmp/hyperfine.deb \ + && rm -fr /tmp/hyperfine.deb + +# Install SonarScanner CLI. +ARG SONAR_SCANNER_HOME=/opt/sonar-scanner +ENV SONAR_SCANNER_HOME=${SONAR_SCANNER_HOME} \ + SONAR_USER_HOME=${SONAR_SCANNER_HOME}/.sonar \ + PATH=${SONAR_SCANNER_HOME}/bin:${PATH} +RUN curl -fsSL https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${sonarScannerVersion}.zip --output sonar-scanner-cli.zip \ + && unzip sonar-scanner-cli.zip \ + && mv sonar-scanner-${sonarScannerVersion} ${SONAR_SCANNER_HOME} \ + && mkdir -p "${SONAR_USER_HOME}" "${SONAR_USER_HOME}/cache" \ + && chown -R ${user}:${user} "${SONAR_SCANNER_HOME}" \ + && chmod -R 777 "${SONAR_USER_HOME}" + +# Set the user's password. +RUN echo "$user:$user" | chpasswd \ + # Grant passwordless sudo access to all users in the 'sudo' group. + # This is required for certain development container features, such as + # Docker-in-Docker, to install and function correctly without user + # intervention. + # While this introduces a slight security risk, it is acceptable in a + # controlled development environment + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# Switch to the non-root user. +USER $user + +# Install pnpm. +RUN corepack install --global pnpm@"${pnpmVersion}" + +# Install Poetry and Pipenv, and preinstall one or more Python versions. +ENV PYENV_ROOT /home/${user}/.pyenv +ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +RUN pipx ensurepath \ + && pipx install \ + poetry=="${poetryVersion}" \ + pipenv=="${pipenvVersion}" \ + uv=="${uvVersion}" \ + && curl -fsSL https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash - \ + && pyenv install ${pyenvPythonVersions} + +# Prepare the user's environment. +RUN usermod --shell /bin/bash $user \ + && printf "%s\n" \ + "" \ + "eval \"\$(pyenv init --path)\"" \ + "eval \"\$(pyenv virtualenv-init -)\"" \ + "" \ + "# source dev-env.sh if found in the current directory" \ + "if [ -f dev-env.sh ]; then" \ + " . ./dev-env.sh" \ + " workspace-initialize-env" \ + "fi" \ + "" | tee -a "/home/$user/.bashrc" diff --git a/.github/.devcontainer/README.md b/.github/.devcontainer/README.md new file mode 100644 index 000000000..4c543aa23 --- /dev/null +++ b/.github/.devcontainer/README.md @@ -0,0 +1,29 @@ +# Sage Monorepo Monorepo Dev Container Prebuild Image + +## Build the image + +From the root workspace: + +```console +devcontainer build \ + --workspace-folder .github \ + --image-name ghcr.io/sage-bionetworks/sage-monorepo-devcontainer:local +``` + +## Start the dev container + +```console +devcontainer up --workspace-folder .github +``` + +## Connect to the dev container + +```console +docker exec -it sage-monorepo-devcontainer-prebuild bash +``` + +## Delete the dev container + +```console +docker rm -f sage-monorepo-devcontainer-prebuild +``` diff --git a/.github/.devcontainer/devcontainer.json b/.github/.devcontainer/devcontainer.json new file mode 100644 index 000000000..11f122da6 --- /dev/null +++ b/.github/.devcontainer/devcontainer.json @@ -0,0 +1,27 @@ +{ + "name": "Sage Monorepo Dev Container Prebuild", + "build": { + "dockerfile": "Dockerfile", + "args": { + // The value of DEVCONTAINER_VERSION will be the commit SHA + // used to prebuild the development container. + "devcontainerVersion": "${localEnv:DEVCONTAINER_VERSION}" + } + }, + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": { + "version": "27.4.1" + }, + "ghcr.io/devcontainers/features/go:1.3.1": { + "version": "1.23", + "golangciLintVersion": "1.63.4" + }, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1.2.0": { + "version": "1.32", + "helm": "3.16.4", + "minikube": "1.34.0" + } + }, + "remoteUser": "ubuntu", + "runArgs": ["--name", "sage-monorepo-devcontainer-prebuild"] +} diff --git a/.github/workflows/build-and-push-devcontainer-image.yml b/.github/workflows/build-and-push-devcontainer-image.yml new file mode 100644 index 000000000..3e6f3c2b0 --- /dev/null +++ b/.github/workflows/build-and-push-devcontainer-image.yml @@ -0,0 +1,221 @@ +name: Dev Container Image + +on: + push: + branches: + - 'main' + paths: + - '.github/workflows/build-and-push-devcontainer-image.yml' + - '.github/.devcontainer/devcontainer.json' + - '.github/.devcontainer/Dockerfile' + pull_request: + paths: + - '.github/workflows/build-and-push-devcontainer-image.yml' + - '.github/.devcontainer/devcontainer.json' + - '.github/.devcontainer/Dockerfile' + +env: + WORKFLOW_FILE: .github/workflows/build-and-push-devcontainer-image.yml + +jobs: + # Lint the Dockerfile using Hadolint in a separate job to ensure code quality + # and prevent unintentional modifications by third-party tools. + lint: + runs-on: ubuntu-24.04 + permissions: + contents: read + env: + # The path to the Dockerfile used for building the development container. + DOCKERFILE: .github/.devcontainer/Dockerfile + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + with: + dockerfile: ${{ env.DOCKERFILE }} + + build-and-push: + runs-on: ubuntu-24.04 + needs: lint + permissions: + # Write access to `contents` needed to upload SBOM to GitHub's dependency graph. + contents: write + packages: write + env: + # The path to the folder containing the `.devcontainer/` directory. + DEVCONTAINER_WORKSPACE_FOLDER: .github + # The name of the image without a tag. + IMAGE_NAME: 'ghcr.io/${{ github.repository }}-devcontainer' + # The path to the Dockerfile used for building the development container. + DOCKERFILE: .github/.devcontainer/Dockerfile + outputs: + # The specific image tagged with the SHA, used for commands targeting a single image version. + image: ${{ steps.meta_sha.outputs.tags }} + # The image digest, available only after pushing the image to the registry. + image_digest: ${{ steps.push.outputs.image_digest }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=edge,branch=main + type=sha,format=long + + - name: Docker meta (sha) + # Generate metadata for Docker images using the commit SHA as the tag. + # The SHA is always available and unique per commit, ensuring traceability. + # Used for steps in the workflow that require a single, unique tag. + id: meta_sha + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + + - name: Securely pull the base image with Docker Content Trust + # DOCKER_CONTENT_TRUST=1 ensures image signatures are verified, but it + # only works for images that have been signed using Docker Content Trust. + # Unsigned images will fail to pull. + run: | + base_image=$(grep '^FROM' "${{ env.DOCKERFILE }}" | tail -n 1 | awk '{print $2}') + DOCKER_CONTENT_TRUST=1 docker pull "${base_image}" + + - name: Build the dev container image + id: build + env: + IMAGES: ${{ steps.meta.outputs.tags }} + run: | + # Export the devcontainer version, which will be set inside the image. See + # devcontainer.json used to build the image. + export DEVCONTAINER_VERSION="${GITHUB_SHA}" + + # Convert the comma-separated list into the desired format + images="" + for image in ${IMAGES}; do + images+="--image-name ${image} " + done + + # Build the image + npm install -g @devcontainers/cli@0.72.0 \ + --integrity="sha512-vDv33/I5POw1wDJmcMbOCTWd3xTk4bbVruJ9Qgr5eiLSl1OsfufN5WfeTZqgK1HeqrNqtH/xPyCKB2LXDNIv3w==" + devcontainer build \ + --workspace-folder "${DEVCONTAINER_WORKSPACE_FOLDER}" \ + ${images} + + - name: Push the Docker image to GitHub Container Registry + if: ${{ github.event_name != 'pull_request' }} + id: push + env: + IMAGES: ${{ steps.meta.outputs.tags }} + run: | + first_iteration=true + for image in ${IMAGES}; do + docker push "${image}" + + # For the first image only, capture the digest + if [ "${first_iteration}" = true ]; then + digest=$(docker inspect --format='{{index .RepoDigests 0}}' "${image}" | cut -d '@' -f2) + echo "image_digest=${digest}" >> "$GITHUB_OUTPUT" + first_iteration=false + fi + done + + - name: Generate SBOM for Docker image + uses: anchore/sbom-action@df80a981bc6edbc4e220a492d3cbe9f5547a6e75 # v0.17.9 + env: + IMAGE: ${{ steps.meta_sha.outputs.tags }} + with: + image: ${{ env.IMAGE }} + format: spdx-json + artifact-name: sbom.spdx.json + upload-artifact: true + dependency-snapshot: ${{ github.event_name != 'pull_request' }} + + cosign: + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-24.04 + needs: build-and-push + permissions: + packages: write + # This is used to complete the identity challenge with sigstore/fulcio + # when running outside of PRs. + id-token: write + steps: + - name: Install Cosign + uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Sign the Docker image with GitHub OIDC Token + env: + IMAGE: ${{ needs.build-and-push.outputs.image }} + IMAGE_DIGEST: ${{ needs.build-and-push.outputs.image_digest }} + run: | + set -x + cosign sign --yes ${IMAGE}@${IMAGE_DIGEST} + + - name: Verify the signature + run: | + set -x + cosign verify \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/${{ github.repository }}/${WORKFLOW_FILE}@${{ github.ref }}" \ + ${{ needs.build-and-push.outputs.image }} + + - name: Download the SBOM artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: sbom.spdx.json + + - name: Create SBOM attestation with Cosign + env: + IMAGE: ${{ needs.build-and-push.outputs.image }} + IMAGE_DIGEST: ${{ needs.build-and-push.outputs.image_digest }} + run: | + cosign attest --yes \ + --type spdxjson \ + --predicate sbom.spdx.json \ + ${IMAGE}@${IMAGE_DIGEST} + + - name: Verify the attestation + run: | + set -x + # Redirect stdout to null to prevent the command from hanging randomly. + # See https://github.com/sigstore/cosign/issues/3602 + cosign verify-attestation \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity "https://github.com/${{ github.repository }}/${WORKFLOW_FILE}@${{ github.ref }}" \ + --type spdxjson \ + ${{ needs.build-and-push.outputs.image }} \ + 1>/dev/null + + - name: Get the signed SBOM + run: | + set -x + cosign download attestation \ + ${{ needs.build-and-push.outputs.image }} \ + | jq -r .payload | base64 -d \ + | jq .predicate