diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 0000000..04c85c0 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Export secrets for Docker containers. +# Restrict exposing secrets only to the steps that need them +export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") +export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") +export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") +export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") +export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") +export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") +export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") +export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") +export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") +export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..4c799f0 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,73 @@ +steps: + - label: ":postgres: Run Tests - Postgres" + key: "run-dbt-postgres" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_POSTGRES_DBT_DBNAME" + - "CI_POSTGRES_DBT_HOST" + - "CI_POSTGRES_DBT_PASS" + - "CI_POSTGRES_DBT_USER" + commands: | + bash .buildkite/scripts/run_models.sh postgres + + - label: ":snowflake-db: Run Tests - Snowflake" + key: "run_dbt_snowflake" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_SNOWFLAKE_DBT_ACCOUNT" + - "CI_SNOWFLAKE_DBT_DATABASE" + - "CI_SNOWFLAKE_DBT_PASS" + - "CI_SNOWFLAKE_DBT_ROLE" + - "CI_SNOWFLAKE_DBT_USER" + - "CI_SNOWFLAKE_DBT_WAREHOUSE" + commands: | + bash .buildkite/scripts/run_models.sh snowflake + + - label: ":gcloud: Run Tests - BigQuery" + key: "run_dbt_bigquery" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "GCLOUD_SERVICE_KEY" + commands: | + bash .buildkite/scripts/run_models.sh bigquery + + - label: ":amazon-redshift: Run Tests - Redshift" + key: "run_dbt_redshift" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_REDSHIFT_DBT_DBNAME" + - "CI_REDSHIFT_DBT_HOST" + - "CI_REDSHIFT_DBT_PASS" + - "CI_REDSHIFT_DBT_USER" + commands: | + bash .buildkite/scripts/run_models.sh redshift + + - label: ":bricks: Run Tests - Databricks" + key: "run_dbt_databricks" + plugins: + - docker#v3.13.0: + image: "python:3.8" + shell: [ "/bin/bash", "-e", "-c" ] + environment: + - "BASH_ENV=/tmp/.bashrc" + - "CI_DATABRICKS_DBT_HOST" + - "CI_DATABRICKS_DBT_HTTP_PATH" + - "CI_DATABRICKS_DBT_TOKEN" + commands: | + bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh new file mode 100644 index 0000000..ea3de4b --- /dev/null +++ b/.buildkite/scripts/run_models.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +apt-get update +apt-get install libsasl2-dev + +python3 -m venv venv +. venv/bin/activate +pip install --upgrade pip setuptools +pip install -r integration_tests/requirements.txt +mkdir -p ~/.dbt +cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml + +db=$1 +echo `pwd` +cd integration_tests +dbt deps +dbt seed --target "$db" --full-refresh +dbt run --target "$db" --full-refresh +dbt test --target "$db" \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 401628d..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2 - -jobs: - build: - docker: - - image: circleci/python:3.7.9-stretch - - steps: - - checkout - - - run: - run: setup_creds - command: | - echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcloud-service-key.json - - run: - name: "Setup dbt" - command: | - sudo apt install libsasl2-dev - python3 -m venv venv - . venv/bin/activate - pip install --upgrade pip setuptools - pip install -r integration_tests/requirements.txt - mkdir -p ~/.dbt - cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - - run: - name: "Run Tests - Postgres" - command: | - . venv/bin/activate - echo `pwd` - cd integration_tests - dbt deps - dbt seed --target postgres --full-refresh - dbt run --target postgres --full-refresh - dbt test --target postgres - - run: - name: "Run Tests - Spark" - command: | - . venv/bin/activate - echo `pwd` - cd integration_tests - dbt deps - dbt seed --target spark --full-refresh - dbt run --target spark --full-refresh - dbt test --target spark - - run: - name: "Run Tests - Redshift" - command: | - . venv/bin/activate - echo `pwd` - cd integration_tests - dbt deps - dbt seed --target redshift --full-refresh - dbt run --target redshift --full-refresh - dbt test --target redshift - - run: - name: "Run Tests - Snowflake" - command: | - . venv/bin/activate - echo `pwd` - cd integration_tests - dbt deps - dbt seed --target snowflake --full-refresh - dbt run --target snowflake --full-refresh - dbt test --target snowflake - - run: - name: "Run Tests - BigQuery" - environment: - GCLOUD_SERVICE_KEY_PATH: "/home/circleci/gcloud-service-key.json" - - command: | - . venv/bin/activate - echo `pwd` - cd integration_tests - dbt deps - dbt seed --target bigquery --full-refresh - dbt run --target bigquery --full-refresh - dbt test --target bigquery \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f450926..50ee0df 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -28,9 +28,9 @@ **How did you test the PR changes?** - + -- [ ] CircleCi +- [ ] Buildkite - [ ] Local (please provide additional testing details below) **Select which warehouse(s) were used to test the PR** diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ff5495..0937297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# dbt_google_ads_source v0.8.1 + +## Updates +- Added `'databricks'` in `src_google_ads.yml` for database configs in order to be compatible for an earlier release of the dbt-databricks adapter. [#32](https://github.com/fivetran/dbt_google_ads_source/pull/32) +- Updated `README.md` to reflect `dbt-labs/spark_utils` in dependency matrix. [#32](https://github.com/fivetran/dbt_google_ads_source/pull/32) + # dbt_google_ads_source v0.8.0 ## 🚨 Breaking Changes 🚨 - The `adwords` api version of the package has been fully removed. As the Fivetran Google Ads connector now requires the Google Ads API, this functionality is no longer used. ([#29](https://github.com/fivetran/dbt_google_ads_source/pull/29)) diff --git a/README.md b/README.md index 00b2cda..e1ba07e 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,15 @@ This dbt package is dependent on the following dbt packages. Please be aware tha > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. ```yml packages: - - package: fivetran/fivetran_utils - version: [">=0.3.0", "<0.4.0"] + - package: fivetran/fivetran_utils + version: [">=0.3.0", "<0.4.0"] + + - package: dbt-labs/dbt_utils + version: [">=0.8.0", "<0.9.0"] + + - package: dbt-labs/spark_utils + version: [">=0.3.0", "<0.4.0"] - - package: dbt-labs/dbt_utils - version: [">=0.8.0", "<0.9.0"] ``` # 🙌 How is this package maintained and can I contribute? diff --git a/dbt_project.yml b/dbt_project.yml index a2c73c5..ee77b65 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'google_ads_source' -version: '0.8.0' +version: '0.8.1' config-version: 2 require-dbt-version: [">=1.0.0", "<2.0.0"] diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index e992ab7..2795e92 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the Google Ads integrations tests with CircleCI. +# HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) config: @@ -7,7 +7,7 @@ config: use_colors: True integration_tests: - target: snowflake + target: redshift outputs: redshift: type: redshift @@ -20,11 +20,11 @@ integration_tests: threads: 8 bigquery: type: bigquery - method: service-account - keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}" + method: service-account-json project: 'dbt-package-testing' schema: google_ads_source_integration_tests threads: 8 + keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: type: snowflake account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" @@ -35,26 +35,20 @@ integration_tests: warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" schema: google_ads_source_integration_tests threads: 8 - spark: - type: spark - method: http - schema: google_ads_source_integration_tests - host: "{{ env_var('CI_SPARK_DBT_HOST') }}" - organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}" - token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}" - cluster: "{{ env_var('CI_SPARK_DBT_CLUSTER') }}" - port: 443 - connect_timeout: 60 - connect_retries: 5 - threads: 4 postgres: type: postgres host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - password: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" + pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" + dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - dbname: "{{ env_var('CI_POSTGRES_DBT_DATABASE') }}" schema: google_ads_source_integration_tests threads: 8 - keepalives_idle: 0 - sslmode: prefer \ No newline at end of file + databricks: + catalog: null + host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" + http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" + schema: google_ads_source_integration_tests + threads: 2 + token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" + type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c96ca0d..f0c7c66 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'google_ads_source_integration_tests' -version: '0.8.0' +version: '0.8.1' profile: 'integration_tests' config-version: 2 diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 5c612de..ceb6b14 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,6 +1,7 @@ -dbt-snowflake~=1.0.0 -dbt-bigquery~=1.0.0 -dbt-redshift~=1.0.0 -dbt-postgres~=1.0.0 -dbt-spark~=1.0.0 -dbt-spark[PyHive]~=1.0.0 \ No newline at end of file +dbt-snowflake>=1.0.0,<1.3.0 +dbt-bigquery>=1.0.0,<1.3.0 +dbt-redshift>=1.0.0,<1.3.0 +dbt-postgres>=1.0.0,<1.3.0 +dbt-spark>=1.0.0,<1.3.0 +dbt-spark[PyHive]>=1.0.0,<1.3.0 +dbt-databricks>=1.0.0,<1.3.0 \ No newline at end of file diff --git a/models/src_google_ads.yml b/models/src_google_ads.yml index 8239159..87a0ab7 100644 --- a/models/src_google_ads.yml +++ b/models/src_google_ads.yml @@ -3,7 +3,7 @@ version: 2 sources: - name: google_ads schema: "{{ var('google_ads_schema', 'google_ads') }}" - database: "{% if target.type != 'spark'%}{{ var('google_ads_database', target.database) }}{% endif %}" + database: "{% if target.type not in ['spark', 'databricks'] %}{{ var('google_ads_database', target.database) }}{% endif %}" loader: Fivetran loaded_at_field: _fivetran_synced