From 884a94ed29e245b6405a849877b0cbd5a0e836a9 Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Fri, 8 Apr 2022 15:17:04 -0700 Subject: [PATCH] Un-Revert OSS branch build for Cloud workflow (#11808) * Revert "Revert "Build OSS branch for deploying to Cloud env (#11474)"" This reverts commit 55e3c1e051bd8dcfa7c09a691dbcfb7901569134. * add action to get dev branch tag to OSS project instead of doing it in cloud * remove dev branch version action, going to do this in cloud afterall --- .../actions/build-and-push-branch/action.yml | 63 +++++++++++++++++++ airbyte-bootloader/Dockerfile | 8 ++- airbyte-bootloader/build.gradle | 2 +- airbyte-cli/build.gradle | 2 +- .../commons/version/AirbyteVersion.java | 12 ++-- airbyte-config/init/build.gradle | 2 +- airbyte-container-orchestrator/Dockerfile | 9 ++- airbyte-container-orchestrator/build.gradle | 2 +- airbyte-db/lib/build.gradle | 2 +- .../integrations/base/IntegrationRunner.java | 4 +- airbyte-metrics/reporter/Dockerfile | 8 ++- airbyte-metrics/reporter/build.gradle | 2 +- airbyte-scheduler/app/Dockerfile | 7 ++- airbyte-scheduler/app/build.gradle | 2 +- airbyte-server/Dockerfile | 7 ++- airbyte-server/build.gradle | 2 +- airbyte-temporal/build.gradle | 2 +- airbyte-webapp/build.gradle | 4 +- airbyte-workers/Dockerfile | 7 ++- airbyte-workers/build.gradle | 2 +- build.gradle | 15 ++++- docker-compose-cloud.build.yaml | 34 ++++++++++ .../developing-locally.md | 6 ++ .../developing-on-docker.md | 6 +- 24 files changed, 171 insertions(+), 39 deletions(-) create mode 100644 .github/actions/build-and-push-branch/action.yml create mode 100644 docker-compose-cloud.build.yaml diff --git a/.github/actions/build-and-push-branch/action.yml b/.github/actions/build-and-push-branch/action.yml new file mode 100644 index 000000000000..80f2d4109854 --- /dev/null +++ b/.github/actions/build-and-push-branch/action.yml @@ -0,0 +1,63 @@ +name: "Build OSS Branch and Push Minimum Required OSS Images" +description: "Build jars and docker images tagged for a particular branch. Primarily used for running OSS branch code in Cloud." +inputs: + branch_version_tag: + description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-" to pass AirbyteVersion validation)' + required: false + dockerhub_token: + description: "Used to log in to dockerhub for pushing images" + required: true +runs: + using: "composite" + steps: + - name: "Parse Input" + id: parse-input + shell: bash + run: |- + # if the *branch_version_tag* input param is not specified, then generate it as 'dev-` + # + [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \ + || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; } + + - uses: actions/setup-java@v1 + with: + java-version: "17" + + - uses: actions/setup-node@v1 + with: + node-version: "16.13.0" + + - name: Set up CI Gradle Properties + run: | + mkdir -p ~/.gradle/ + cat > ~/.gradle/gradle.properties < connectorClas final String version = parseConnectorVersion(env.getOrDefault("WORKER_CONNECTOR_IMAGE", "")); final String airbyteVersion = env.getOrDefault(EnvConfigs.AIRBYTE_VERSION, ""); final String airbyteRole = env.getOrDefault(EnvConfigs.AIRBYTE_ROLE, ""); - final boolean isDev = version.equals(AirbyteVersion.DEV_VERSION) - || airbyteVersion.equals(AirbyteVersion.DEV_VERSION) + final boolean isDev = version.startsWith(AirbyteVersion.DEV_VERSION_PREFIX) + || airbyteVersion.startsWith(AirbyteVersion.DEV_VERSION_PREFIX) || airbyteRole.equals("airbyter"); if (isDev) { LOGGER.debug("Skip Sentry transaction for dev environment"); diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index ceccfc953f29..356b4c47b689 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -1,11 +1,15 @@ ARG JDK_VERSION=17.0.1 FROM openjdk:${JDK_VERSION}-slim AS metrics-reporter +ARG VERSION=0.35.65-alpha + ENV APPLICATION airbyte-metrics-reporter +ENV VERSION ${VERSION} WORKDIR /app -ADD bin/${APPLICATION}-0.35.65-alpha.tar /app +ADD bin/${APPLICATION}-${VERSION}.tar /app + # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-metrics/reporter/build.gradle b/airbyte-metrics/reporter/build.gradle index ec1e02fccff0..31cfbb425b15 100644 --- a/airbyte-metrics/reporter/build.gradle +++ b/airbyte-metrics/reporter/build.gradle @@ -25,6 +25,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("metrics-reporter", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-scheduler/app/Dockerfile b/airbyte-scheduler/app/Dockerfile index 609608392c23..bf0ea81e55ca 100644 --- a/airbyte-scheduler/app/Dockerfile +++ b/airbyte-scheduler/app/Dockerfile @@ -1,11 +1,14 @@ ARG JDK_VERSION=17.0.1 FROM openjdk:${JDK_VERSION}-slim AS scheduler +ARG VERSION=0.35.65-alpha + ENV APPLICATION airbyte-scheduler +ENV VERSION ${VERSION} WORKDIR /app -ADD bin/${APPLICATION}-0.35.65-alpha.tar /app +ADD bin/${APPLICATION}-${VERSION}.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-scheduler/app/build.gradle b/airbyte-scheduler/app/build.gradle index 46d842a45e46..13953e969907 100644 --- a/airbyte-scheduler/app/build.gradle +++ b/airbyte-scheduler/app/build.gradle @@ -65,6 +65,6 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("scheduler", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("scheduler", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 3fefc03f30ea..0c6c60ac8510 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,11 +3,14 @@ FROM openjdk:${JDK_VERSION}-slim AS server EXPOSE 8000 +ARG VERSION=0.35.65-alpha + ENV APPLICATION airbyte-server +ENV VERSION ${VERSION} WORKDIR /app -ADD bin/${APPLICATION}-0.35.65-alpha.tar /app +ADD bin/${APPLICATION}-${VERSION}.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 8d0b77e1c16e..e4e8d26966ba 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -145,7 +145,7 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("server", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-temporal/build.gradle b/airbyte-temporal/build.gradle index c9de2669c1d5..14a51ac94221 100644 --- a/airbyte-temporal/build.gradle +++ b/airbyte-temporal/build.gradle @@ -5,6 +5,6 @@ task copyScripts(type: Copy) { into 'build/docker/bin/scripts' } -Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("temporal", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyScripts) assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index c4b3123dbeab..44037bdb411e 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -82,9 +82,9 @@ copyAssets.dependsOn npm_run_build assemble.dependsOn copyDocs copyDocker.dependsOn(npm_run_build) -Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("webapp", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyBuild) dockerBuildTask.dependsOn(copyNginx) dockerBuildTask.dependsOn(copyDocs) dockerBuildTask.dependsOn(copyAssets) -assemble.dependsOn(dockerBuildTask) \ No newline at end of file +assemble.dependsOn(dockerBuildTask) diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 7aedade8472a..36cf338ee4d5 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -25,12 +25,15 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl +ARG VERSION=0.35.65-alpha + ENV APPLICATION airbyte-workers +ENV VERSION ${VERSION} WORKDIR /app # Move worker app -ADD bin/${APPLICATION}-0.35.65-alpha.tar /app +ADD bin/${APPLICATION}-${VERSION}.tar /app # wait for upstream dependencies to become available before starting server -ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-0.35.65-alpha/bin/${APPLICATION}"] +ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"] diff --git a/airbyte-workers/build.gradle b/airbyte-workers/build.gradle index 81a28b4e35cd..0091663a364a 100644 --- a/airbyte-workers/build.gradle +++ b/airbyte-workers/build.gradle @@ -76,7 +76,7 @@ task copyGeneratedTar(type: Copy) { into 'build/docker/bin' } -Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir") +Task dockerBuildTask = getDockerBuildTask("worker", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag") dockerBuildTask.dependsOn(copyGeneratedTar) assemble.dependsOn(dockerBuildTask) diff --git a/build.gradle b/build.gradle index a2a1fee6c0ae..9eab42684434 100644 --- a/build.gradle +++ b/build.gradle @@ -30,6 +30,15 @@ if (!env.containsKey('VERSION')) { throw new Exception('Version not specified in .env file...') } +// `version` is used as the application build version for artifacts like jars +// `image_tag` is used as the docker tag applied to built images. +// These values are the same for building an specific Airbyte release or branch via the 'VERSION' environment variable. +// For local development builds, the 'VERSION' environment variable is unset, and built images are tagged with 'dev'. +ext { + version = System.getenv("VERSION") ?: env.VERSION + image_tag = System.getenv("VERSION") ?: 'dev' +} + def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython -> /* In java, we don't have a second linter/styling tool other than spotless so it doesn't really @@ -129,9 +138,8 @@ spotless { check.dependsOn 'spotlessApply' @SuppressWarnings('GroovyAssignabilityCheck') -def Task getDockerBuildTask(String artifactName, String projectDir) { +def Task getDockerBuildTask(String artifactName, String projectDir, String buildVersion, String buildTag) { return task ("buildDockerImage-$artifactName"(type: DockerBuildImage) { - def buildTag = System.getenv('VERSION') ?: 'dev' def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.1' def arch = System.getProperty("os.arch").toLowerCase() @@ -149,6 +157,7 @@ def Task getDockerBuildTask(String artifactName, String projectDir) { buildArgs.put('DOCKER_BUILD_ARCH', buildArch) buildArgs.put('ALPINE_IMAGE', alpineImage) buildArgs.put('POSTGRES_IMAGE', postgresImage) + buildArgs.put('VERSION', buildVersion) }) } @@ -170,7 +179,7 @@ allprojects { group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}" project.archivesBaseName = "${project.group}-${project.name}" - version = env.VERSION + version = rootProject.ext.version } // Java projects common configurations diff --git a/docker-compose-cloud.build.yaml b/docker-compose-cloud.build.yaml new file mode 100644 index 000000000000..0ec5bfaee1fb --- /dev/null +++ b/docker-compose-cloud.build.yaml @@ -0,0 +1,34 @@ +# Defines the minimum set of images needed to run Cloud. +# Used to push OSS images that Cloud depends on. + +version: "3.7" + +services: + scheduler: + image: airbyte/scheduler:${VERSION} + build: + dockerfile: Dockerfile + context: airbyte-scheduler/app + labels: + io.airbyte.git-revision: ${GIT_REVISION} + worker: + image: airbyte/worker:${VERSION} + build: + dockerfile: Dockerfile + context: airbyte-workers + labels: + io.airbyte.git-revision: ${GIT_REVISION} + webapp: + image: airbyte/webapp:${VERSION} + build: + dockerfile: Dockerfile + context: airbyte-webapp + labels: + io.airbyte.git-revision: ${GIT_REVISION} + metric-reporter: + image: airbyte/metrics-reporter:${VERSION} + build: + dockerfile: Dockerfile + context: airbyte-metrics/reporter + labels: + io.airbyte.git-revision: ${GIT_REVISION} diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/developing-locally.md index 947bce8842f0..feeccd5f6ed5 100644 --- a/docs/contributing-to-airbyte/developing-locally.md +++ b/docs/contributing-to-airbyte/developing-locally.md @@ -40,6 +40,12 @@ This will build all the code and run all the unit tests. `SUB_BUILD=PLATFORM ./gradlew build` creates all the necessary artifacts \(Webapp, Jars and Docker images\) so that you can run Airbyte locally. Since this builds everything, it can take some time. +{% hint style="info" %} +Optionally, you may pass a `VERSION` environment variable to the gradle build command. If present, gradle will use this value as a tag for all created artifacts (both Jars and Docker images). + +If unset, gradle will default to using the current VERSION in `.env` for Jars, and `dev` as the Docker image tag. +{% endhint %} + {% hint style="info" %} Gradle will use all CPU cores by default. If Gradle uses too much/too little CPU, tuning the number of CPU cores it uses to better suit a dev's need can help. diff --git a/docs/contributing-to-airbyte/developing-on-docker.md b/docs/contributing-to-airbyte/developing-on-docker.md index 16b9f16cfbf4..babe2447b26f 100644 --- a/docs/contributing-to-airbyte/developing-on-docker.md +++ b/docs/contributing-to-airbyte/developing-on-docker.md @@ -14,13 +14,13 @@ method takes 2 arguments: ## Adding a new docker build Once you have a `Dockerfile`, generating the docker image is done in the following way: -- specify the artifact name and the project directory, +- specify the artifact name, the project directory, the version, and the tag. - make sure that the Dockerfile is properly copied to the docker context dir before building the image - make the build docker task to depend on the `assemble` task. For example: ```groovy -Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("cli", project.projectDir, rootProject.ext.version, rootProject.ext.image_tag) dockerBuildTask.dependsOn(copyDocker) assemble.dependsOn(dockerBuildTask) ``` @@ -37,7 +37,7 @@ task copyScripts(type: Copy) { into 'build/docker/bin/scripts' } -Task dockerBuildTask = getDockerBuildTask("init", project.projectDir) +Task dockerBuildTask = getDockerBuildTask("init", project.projectDir, rootProject.ext.version, rootProject.ext.image_tag) dockerBuildTask.dependsOn(copyScripts) assemble.dependsOn(dockerBuildTask) ```