diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml new file mode 100644 index 0000000..52f4563 --- /dev/null +++ b/.github/workflows/examples.yml @@ -0,0 +1,126 @@ +name: Test sdf-action + +on: + pull_request: + +jobs: + tpch_redshift: + runs-on: ubuntu-latest + name: Redshift workspace + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: sdf compile step + uses: ./ # Uses an action in the root directory + id: sdf + with: + command: 'sdf compile --show all' + workspace_dir: 'workspace/tpch_redshift' + + aws_region: 'us-west-2' + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # Use the output from the `sdf` step + - name: Display the sdf output + run: | + echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + + # dbt_init_challenge_v1 - uses dbt-action + dbt_init_challenge_v1: + runs-on: ubuntu-latest + name: Snowflake + dbt workspace - run dbt commands using dbt-action + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run DBT commands + uses: mwhitaker/dbt-action@master + env: + DBT_TARGET: dev + SNOWFLAKE_ACCOUNT_ID: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + with: + dbt_command: "dbt compile" + dbt_project_folder: "workspace/dbt_init_challenge" + + - name: Run sdf compile + uses: ./ # Uses an action in the root directory + id: sdf + env: + CATALOG: dbt_hol_dev + SCHEMA: public + with: + command: 'sdf compile' + workspace_dir: 'workspace/dbt_init_challenge' + access_key: ${{ secrets.ACCESS_KEY }} + secret_key: ${{ secrets.SECRET_KEY }} + + is_dbt: 'true' + snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }} + snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }} + snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }} + snowflake_role: 'dbt_test_role' + snowflake_warehouse: 'dbt_dev_wh' + + # Use the output from the `sdf` step + - name: Display the sdf output + run: | + echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + + # dbt_init_challenge_v2: rus dbt commands in a container job + dbt_init_challenge_v2: + container: + image: ghcr.io/dbt-labs/dbt-snowflake:1.5.latest + volumes: + - ${{ github.workspace }}:/repo + + runs-on: ubuntu-latest + name: Snowflake + dbt workspace - run dbt commands in a container job + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run DBT commands + env: + DBT_TARGET: dev + SNOWFLAKE_ACCOUNT_ID: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }} + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + run: | + cd /repo/workspace/dbt_init_challenge + dbt deps + dbt compile + - name: Run sdf push + uses: ./ # Uses an action in the root directory + id: sdf + env: + CATALOG: dbt_hol_dev + SCHEMA: public + with: + command: 'sdf push' + # relative path to ${{ github.workspace }} which is automatically mounted by GitHub Actions + workspace_dir: workspace/dbt_init_challenge + access_key: ${{ secrets.ACCESS_KEY }} + secret_key: ${{ secrets.SECRET_KEY }} + + is_dbt: 'true' + snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }} + snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }} + snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }} + snowflake_role: 'dbt_test_role' + snowflake_warehouse: 'dbt_dev_wh' + + # Use the output from the `sdf` step + - name: Display the sdf output + run: | + echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index add3968..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Test sdf-action - -on: - pull_request: - -jobs: - dbt_init_challenge_job: - runs-on: ubuntu-latest - name: Run sdf push on a DBT + Snowflake workspace - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: sdf push step - uses: ./ # Uses an action in the root directory - id: sdf - with: - command: 'sdf push' - workspace_dir: 'workspace/dbt_init_challenge' - access_key: ${{ secrets.ACCESS_KEY }} - secret_key: ${{ secrets.SECRET_KEY }} - - is_dbt: 'true' - snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }} - snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }} - snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }} - snowflake_role: 'dbt_test_role' - snowflake_warehouse: 'dbt_dev_wh' - dbt_target: 'dev' - - # Use the output from the `sdf` step - - name: Display the sdf output - run: | - echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - - tpch_redshift: - runs-on: ubuntu-latest - name: Run sdf compile on a Redshift workspace - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: sdf compile step - uses: ./ # Uses an action in the root directory - id: sdf - with: - command: 'sdf compile --show all' - workspace_dir: 'workspace/tpch_redshift' - - aws_region: 'us-west-2' - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - # Use the output from the `sdf` step - - name: Display the sdf output - run: | - echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..db0478f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,2 @@ +# Contributing to sdf-action + diff --git a/Dockerfile b/Dockerfile index 2c8d7c2..12aa49d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,11 @@ FROM python:3.12.2-bookworm -ARG SDF_VERSION=0.1.192 # Install dependencies RUN apt-get update && apt-get install -y \ - yq \ + jq \ curl \ && rm -rf /var/lib/apt/lists/* -# Install sdf -RUN curl -LSfs https://cdn.sdf.com/releases/download/install.sh | bash -s -- --version ${SDF_VERSION} - -# Install dbt-snowflake plugin using pip -RUN python3.12 -m venv .venv && . .venv/bin/activate \ - && python3 -m pip install dbt-snowflake==1.7.2 \ - && python3 -m pip install --upgrade dbt-core==1.7.9 \ - && python3 -m pip install protobuf==4.25.3 - -# Copy your code file from your action repository to the filesystem path `/` of the container COPY entrypoint.sh /entrypoint.sh # Set the code file as the entry point diff --git a/Dockerfile.normal b/Dockerfile.normal deleted file mode 100644 index 062b8f6..0000000 --- a/Dockerfile.normal +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.12.2-bookworm -ARG SDF_VERSION=0.1.192 - -# Install dependencies -RUN apt-get update && apt-get install -y \ - yq \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Install sdf -RUN curl -LSfs https://cdn.sdf.com/releases/download/install.sh | bash -s -- --version ${SDF_VERSION} - -# Copy your code file from your action repository to the filesystem path `/` of the container -COPY entrypoint.sh /entrypoint.sh - -# Set the code file as the entry point -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1dca4aa --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright [2024] [SDF Labs] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md index 7a06392..4a4c053 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,29 @@ # sdf-action -[WIP] A Github Action to run `sdf` CLI in workflows. +Github Action to run `sdf` CLI in workflows. + +## Usage +Check out this [example workflow](./.github/workflows/examples.yml) to see how to use this action. + +## Parameters + +* `snowflake_`* parameters are required if the `snowflake` table provider is used +* `aws_*` parameters are required if the `redshift` table provider is used +* `access_key` and `secret_key` are required if the `sdf push` is used in `command` + +| Parameter | Description | Required | Default | +| --- | --- | --- | --- | +| `sdf_version` | SDF CLI version | No | `"0.2.10"` +| `command` | The `sdf` CLI command to run. | No | `sdf compile` +| `workspace_dir` | The directory of the workspace | No | `"."` | +| `access_key` | access key created from the [console](https://console.sdf.com/catalog/settings/general) to be used in `sdf push` | No | | +| `secret_key` | secret key created from the [console](https://console.sdf.com/catalog/settings/general) to be used in `sdf push` | No | | +| `is_dbt` | Set to a non-empty string if the workspace is dbt based | No | `""` | | +| `dbt_target` | The dbt target | No | `"dev"` | +| `snowflake_account_id` | The snowflake account | No | | +| `snowflake_username` | The snowflake username | No | | +| `snowflake_password` | The snowflake password | No | | +| `snowflake_role` | The snowflake role | No | | +| `snowflake_warehouse` | The snowflake warehouse | No | | +| `aws_region` | The aws region | No | | +| `aws_access_key_id` | The aws access key id created from an IAM user | No | | +| `aws_secret_access_key` | The aws secret created from an IAM user | No | | diff --git a/action.yml b/action.yml index 6f36e12..2b3e906 100644 --- a/action.yml +++ b/action.yml @@ -1,15 +1,19 @@ # action.yml -name: "dbt action" +name: "sdf action" description: "Run sdf cli commands" inputs: + sdf_version: + description: "The version of the sdf cli" + default: "0.2.10" + required: false command: description: "The sdf command to run" default: "sdf compile" - required: true + required: false workspace_dir: description: "The directory of the workspace" default: "." - required: true + required: false access_key: description: "The access key" required: false @@ -18,11 +22,12 @@ inputs: required: false is_dbt: description: "Whether the workspace is dbt based or not" - required: true + required: false default: "" dbt_target: description: "The dbt target" required: false + default: "dev" # required if snowflake table provider is used snowflake_account_id: @@ -63,6 +68,7 @@ runs: args: - ${{ inputs.command }} - ${{ inputs.is_dbt }} + - ${{ inputs.sdf_version }} env: WORKSPACE_DIR: ${{ inputs.workspace_dir }} ACCESS_KEY: ${{ inputs.access_key }} diff --git a/entrypoint.sh b/entrypoint.sh index 3a21b3a..ea2bc3e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,13 +1,52 @@ #!/bin/bash -l -source /.venv/bin/activate +input_command=$1 +input_is_dbt=$2 +sdf_version=$3 + +is_less_semvar() { + # split the version strings into arrays + IFS='.' read -ra version1 <<<"$1" + IFS='.' read -ra version2 <<<"$2" + + i=0 + while true; do + v1="${version1[$i]:-0}" + v2="${version2[$i]:-0}" + + # compare numerically + if ((v1 < v2)); then + echo 1 + return + elif ((v1 > v2)); then + echo 0 + return + fi + ((i++)) + + # run out of components to compare + if [[ -z "${version1[$i]}" && -z "${version2[$i]}" ]]; then + echo 0 + return + fi + done +} + +min_version=$(curl https://api.sdf.com/api/v2/public.minVersion | jq -r '.result.data.json') +if [[ $(is_less_semvar $sdf_version $min_version) -eq 1 ]]; then + echo "The input SDF CLI version is lower than the minimum required version: $min_version" + exit 1 +fi + +if [ "$(uname -m)" = "aarch64" ]; then + curl -LSfs https://cdn.sdf.com/releases/download/install.sh | bash -s -- --version ${sdf_version} --target aarch64-unknown-linux-gnu +else + curl -LSfs https://cdn.sdf.com/releases/download/install.sh | bash -s -- --version ${sdf_version} +fi echo "workspace dir set as: \"${WORKSPACE_DIR}\"" cd ${WORKSPACE_DIR} -input_command=$1 -input_is_dbt=$2 - check_exit_status() { exit_status=$1 command_log=$2 @@ -20,7 +59,7 @@ check_exit_status() { echo "$command_log" echo EOF } >>$GITHUB_OUTPUT - + echo "result=failed" >>$GITHUB_OUTPUT exit $exit_status fi @@ -34,33 +73,12 @@ if [[ $input_command == "sdf push"* ]]; then fi if [[ -n $input_is_dbt ]]; then - echo "python3 version - $(python3 --version)" - echo "::group::Setting up dbt" - echo "running dbt deps" - dbt deps - check_exit_status $? "" - - echo "running dbt compile" - dbt compile - check_exit_status $? "" - - echo "running dbt compile done" sdf dbt refresh echo "::endgroup::" check_exit_status $? "" fi -# run sdf auth login snwoflake if necessary -snowflake_provider=$(yq .provider.type workspace.sdf.yml | grep snowflake | tail -1) -if [[ -n $snowflake_provider ]]; then - echo "snowflake provider used: running 'sdf auth login'" - sdf auth login snowflake \ - --account-id "${SNOWFLAKE_ACCOUNT_ID}" --username "${SNOWFLAKE_USERNAME}" --password "${SNOWFLAKE_PASSWORD}" \ - --role "${SNOWFLAKE_ROLE}" --warehouse "${SNOWFLAKE_WAREHOUSE}" - check_exit_status $? "" -fi - # run and save outputs echo "running command: $input_command" log=$($input_command 2>&1) @@ -73,4 +91,4 @@ check_exit_status $exit_status "$log" echo "$log" echo EOF } >>$GITHUB_OUTPUT -echo "result=passed" >>$GITHUB_OUTPUT \ No newline at end of file +echo "result=passed" >>$GITHUB_OUTPUT diff --git a/workspace/dbt_init_challenge/workspace.sdf.yml b/workspace/dbt_init_challenge/workspace.sdf.yml index d950d3a..b93b4f9 100644 --- a/workspace/dbt_init_challenge/workspace.sdf.yml +++ b/workspace/dbt_init_challenge/workspace.sdf.yml @@ -1,15 +1,15 @@ workspace: - edition: "1.1" + edition: "1.2" name: dbt_hol includes: - path: target/compiled/sdf/models index: catalog-schema-table-name - dialect: snowflake - default-catalog: dbt_hol_dev - default-schema: public - default-profile: dbg ---- -provider: - sources: - - census.public.* - type: snowflake + providers: + - namespaces: + - target: census.public.* + type: snowflake + defaults: + environment: dbg + dialect: snowflake + catalog: "${{ env_var('CATALOG') }}" + schema: "${{ env_var('SCHEMA') }}" diff --git a/workspace/tpch_redshift/workspace.sdf.yml b/workspace/tpch_redshift/workspace.sdf.yml index 74fb3dd..b039d2e 100644 --- a/workspace/tpch_redshift/workspace.sdf.yml +++ b/workspace/tpch_redshift/workspace.sdf.yml @@ -1,14 +1,16 @@ workspace: - edition: "1.1" + edition: "1.2" name: "tpch" - default-catalog: sample_data_dev - default-schema: tpch - compute: "local" + defaults: + environment: dbg + dialect: redshift + catalog: sample_data_dev + schema: tpch + compute: "local" includes: - path: queries/ ---- -provider: - type: redshift - cluster-identifier: sdf-cli-tests - sources: - - sample_data_dev + providers: + - type: redshift + cluster-identifier: sdf-cli-tests + namespaces: + - target: sample_data_dev.*.*