Merge pull request #17 from sdf-labs/ani/additional_snowflake_and_bq_…

…auth_support

Add Snowflake Key Pair Auth and BigQuery Credential Support

.github/workflows/examples.yml

@@ -15,10 +15,10 @@ jobs:
         uses: ./ # Uses an action in the root directory
         id: sdf
         with:
-          command: 'sdf compile --show all'
-          workspace_dir: 'workspace/tpch_redshift'
+          command: "sdf compile --show all"
+          workspace_dir: "workspace/tpch_redshift"
 
-          aws_region: 'us-west-2'
+          aws_region: "us-west-2"
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
@@ -29,61 +29,13 @@ jobs:
           echo '```' >>$GITHUB_STEP_SUMMARY
           echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY
           echo '```' >>$GITHUB_STEP_SUMMARY
-  
-  # dbt_init_challenge_v1 - uses dbt-action
-  dbt_init_challenge_v1:
-    runs-on: ubuntu-latest
-    name: Snowflake + dbt workspace - run dbt commands using dbt-action
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Run DBT commands
-        uses: mwhitaker/[email protected]
-        env: 
-          DBT_TARGET: dev
-          SNOWFLAKE_ACCOUNT_ID: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
-          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-        with:
-          dbt_command: "dbt compile"
-          dbt_project_folder: "workspace/dbt_init_challenge"
-
-      - name: Run sdf push
-        uses: ./ # Uses an action in the root directory
-        id: sdf
-        env:
-          CATALOG: dbt_hol_dev
-          SCHEMA: public
-          DBT_TARGET: dev
-        with:
-          # TODO: FIX compile failed
-          command: 'sdf push'
-          workspace_dir: 'workspace/dbt_init_challenge'
-          access_key: ${{ secrets.ACCESS_KEY }}
-          secret_key: ${{ secrets.SECRET_KEY }}
-
-          is_dbt: 'true'
-          snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
-          snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }}
-          snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }}
-          snowflake_role: 'dbt_test_role'
-          snowflake_warehouse: 'dbt_dev_wh'
 
-      # Use the output from the `sdf` step
-      - name: Display the sdf output
-        run: |
-          echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY
-          echo '```' >>$GITHUB_STEP_SUMMARY
-          echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY
-          echo '```' >>$GITHUB_STEP_SUMMARY
-
-  # dbt_init_challenge_v2: rus dbt commands in a container job
-  dbt_init_challenge_v2:
+  dbt_init_challenge:
     container:
-      image: ghcr.io/dbt-labs/dbt-snowflake:1.5.latest
+      image: ghcr.io/dbt-labs/dbt-snowflake:1.8.latest
       volumes:
         - ${{ github.workspace }}:/repo
-    
+
     runs-on: ubuntu-latest
     name: Snowflake + dbt workspace - run dbt commands in a container job
     steps:
@@ -107,19 +59,19 @@ jobs:
           SCHEMA: public
           DBT_TARGET: dev
         with:
-          command: 'sdf push'
+          command: "sdf push"
           # relative path to ${{ github.workspace }} which is automatically mounted by GitHub Actions
           workspace_dir: workspace/dbt_init_challenge
           access_key: ${{ secrets.ACCESS_KEY }}
           secret_key: ${{ secrets.SECRET_KEY }}
 
-          is_dbt: 'true'
+          is_dbt: "true"
           snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
           snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }}
           snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }}
-          snowflake_role: 'dbt_test_role'
-          snowflake_warehouse: 'dbt_dev_wh'
-  
+          snowflake_role: "dbt_test_role"
+          snowflake_warehouse: "dbt_dev_wh"
+
       # Use the output from the `sdf` step
       - name: Display the sdf output
         run: |
@@ -130,10 +82,10 @@ jobs:
 
   snowflake:
     container:
-      image: ghcr.io/dbt-labs/dbt-snowflake:1.5.latest
+      image: ghcr.io/dbt-labs/dbt-snowflake:1.8.latest
       volumes:
         - ${{ github.workspace }}:/repo
-    
+
     runs-on: ubuntu-latest
     name: Snowflake workspace
     steps:
@@ -145,7 +97,7 @@ jobs:
         id: sdf
         with:
           # TODO: FIX compile failed
-          command: 'sdf push'
+          command: "sdf push"
           # relative path to ${{ github.workspace }} which is automatically mounted by GitHub Actions
           workspace_dir: workspace/snowflake
           access_key: ${{ secrets.ACCESS_KEY }}
@@ -154,13 +106,13 @@ jobs:
           snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
           snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }}
           snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }}
-          snowflake_role: 'dbt_test_role'
-          snowflake_warehouse: 'dbt_dev_wh'
-  
+          snowflake_role: "dbt_test_role"
+          snowflake_warehouse: "dbt_dev_wh"
+
       # Use the output from the `sdf` step
       - name: Display the sdf output
         run: |
           echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY
           echo '```' >>$GITHUB_STEP_SUMMARY
           echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY
-          echo '```' >>$GITHUB_STEP_SUMMARY
+          echo '```' >>$GITHUB_STEP_SUMMARY

README.md

@@ -1,17 +1,19 @@
 [![SDF + Snowflake / Redshift / DBT Tests](https://github.com/sdf-labs/sdf-action/actions/workflows/examples.yml/badge.svg)](https://github.com/sdf-labs/sdf-action/actions/workflows/examples.yml)
 
-
 # Official SDF GitHub Action
 
-This action is used to run SDF in CI/CD workflows with GitHub Actions. A common use case would be to run an impact analysis with `sdf compile` and `sdf check` on PR creation. 
+This action is used to run SDF in CI/CD workflows with GitHub Actions. A common use case would be to run an impact analysis with `sdf compile` and `sdf check` on PR creation.
 
 *Note: This GitHub action is still < v1, as such certain scenarios may result in unexpected behavior. Please follow the [contributing guidelines](./CONTRIBUTING.md) and create an issue in this repo if you find any bugs or issues.*
 
 For an in-depth guide on how to use this GitHub, please see the GitHub CI/CD section of [our official docs](https://docs.sdf.com/integrations/cicd/ci_cd)
 
 ## Usage
+
 Check out this [example workflow](./.github/workflows/examples.yml) to see how to use this action.
 
+*Note: make sure to use a `dbt` version that is at least `1.8.0` if you need to use this action for a dbt project*
+
 <!-- start usage -->
 ```yaml
 - uses: sdf-labs/sdf-action@v0
@@ -46,31 +48,53 @@ Check out this [example workflow](./.github/workflows/examples.yml) to see how t
 
     ### Snowflake Integration
 
-    # All of the following are used to authenticate to Snowflake.
-    # All of these are required if using a Snowflake provider.
+    # Snowflake authentication configuration
+    # Required parameters for Snowflake provider:
     snowflake_account_id: ${{ secrets.SNOWFLAKE_ACCOUNT_ID }}
     snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }}
-    snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }}
     snowflake_role: ${{ secrets.SNOWFLAKE_ROLE }}
     snowflake_warehouse: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
 
+    # Authentication method 1: Password-based (choose either method 1 or 2)
+    snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }}
+
+    # Authentication method 2: Key-pair based (choose either method 1 or 2)
+    # Provide either private_key_path OR private_key_pem:
+    snowflake_private_key_path: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }}  # Path to key file
+    # OR
+    snowflake_private_key_pem: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PEM }}    # Direct PEM content
+
+    # Optional for key-pair authentication:
+    snowflake_private_key_passphrase: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
+
     ### AWS / Redshift Integration
 
     # All of the following are required if using an S3, AWS Glue, or AWS Redshift
     # provider.
     aws_region: ${{ secrets.AWS_REGION }}
     aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
     aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    ### BigQuery Integration
+
+    # Authentication method 1: Individual credentials
+    bigquery_project_id: ${{ secrets.BIGQUERY_PROJECT_ID }}
+    bigquery_client_email: ${{ secrets.BIGQUERY_CLIENT_EMAIL }}
+    bigquery_private_key: ${{ secrets.BIGQUERY_PRIVATE_KEY }}
+
+    # OR
+
+    # Authentication method 2: Credentials JSON file
+    bigquery_credentials_json_path: ${{ secrets.BIGQUERY_CREDENTIALS_JSON_PATH }}
 ```
 <!-- end usage -->
 
 # Scenarios
 
-** [Run Compile on PR](#run-compile-on-pr-creation-and-update)
+**[Run Compile on PR](#run-compile-on-pr-creation-and-update)
 ** [Run Checks on PR with Snowflake Provider](#run-compile-on-pr-creation-and-update-with-a-snowflake-provider)
 ** [Push to SDF Cloud on Merge to Main](#push-to-sdf-cloud-on-merge-to-main-with-a-redshift-provider)
 
-
 ## Run Compile on PR Creation and Update
 
 ```yaml
@@ -136,7 +160,7 @@ on:
 | `access_key` | access key created from the [console](https://console.sdf.com/catalog/settings/general) to be used in `sdf push`  | No | |
 | `secret_key` | secret key created from the [console](https://console.sdf.com/catalog/settings/general) to be used in `sdf push` | No | |
 | `is_dbt` | Set to a non-empty string if the workspace is dbt based | No | `""` | |
-| `snowflake_account_id` | The snowflake account | No | | 
+| `snowflake_account_id` | The snowflake account | No | |
 | `snowflake_username` | The snowflake username | No | |
 | `snowflake_password` | The snowflake password | No | |
 | `snowflake_role` | The snowflake role | No | |

action.yml

@@ -39,6 +39,15 @@ inputs:
   snowflake_password:
     description: "The snowflake password"
     required: false
+  snowflake_private_key_path:
+    description: "The path to the private key file"
+    required: false
+  snowflake_private_key_pem:
+    description: "The private key pem"
+    required: false
+  snowflake_private_key_passphrase:
+    description: "The private key passphrase"
+    required: false
   snowflake_role:
     description: "The snowflake role"
     required: false
@@ -57,6 +66,20 @@ inputs:
     description: "The aws secret access key"
     required: false
 
+  # required if bigquery table provider is used
+  bigquery_project_id:
+    description: "The bigquery project id"
+    required: false
+  bigquery_client_email:
+    description: "The bigquery client email"
+    required: false
+  bigquery_private_key:
+    description: "The private key pem"
+    required: false
+  bigquery_credentials_json_path:
+    description: "The path to the bigquery credentials json file"
+    required: false
+
 outputs:
   result:
     description: "The result (pass or failed)"
@@ -80,7 +103,15 @@ runs:
     SNOWFLAKE_PASSWORD: ${{ inputs.snowflake_password }}
     SNOWFLAKE_ROLE: ${{ inputs.snowflake_role }}
     SNOWFLAKE_WAREHOUSE: ${{ inputs.snowflake_warehouse }}
+    SNOWFLAKE_PRIVATE_KEY_PATH: ${{ inputs.snowflake_private_key_path }}
+    SNOWFLAKE_PRIVATE_KEY_PEM: ${{ inputs.snowflake_private_key_pem }}
+    SNOWFLAKE_PRIVATE_KEY_PASSPHRASE: ${{ inputs.snowflake_private_key_passphrase }}
 
     AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
     AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
     AWS_DEFAULT_REGION: ${{ inputs.aws_region }}
+
+    BIGQUERY_PROJECT_ID: ${{ inputs.bigquery_project_id }}
+    BIGQUERY_CLIENT_EMAIL: ${{ inputs.bigquery_client_email }}
+    BIGQUERY_PRIVATE_KEY: ${{ inputs.bigquery_private_key }}
+    BIGQUERY_CREDENTIALS_JSON_PATH: ${{ inputs.bigquery_credentials_json_path }}

entrypoint.sh

@@ -107,9 +107,35 @@ fi
 
 if [ -n "${SNOWFLAKE_ACCOUNT_ID}" ]; then
   echo "snowflake provider used: running 'sdf auth login snowflake'"
-  sdf auth login snowflake \
-    --account-id "${SNOWFLAKE_ACCOUNT_ID}" --username "${SNOWFLAKE_USERNAME}" --password "${SNOWFLAKE_PASSWORD}" \
-    --role "${SNOWFLAKE_ROLE}" --warehouse "${SNOWFLAKE_WAREHOUSE}"
+
+  # Base command with common parameters
+  auth_command="sdf auth login snowflake --account-id \"${SNOWFLAKE_ACCOUNT_ID}\" --username \"${SNOWFLAKE_USERNAME}\""
+
+  if [ -n "${SNOWFLAKE_PASSWORD}" ]; then
+    # Password-based authentication
+    auth_command+=" --password \"${SNOWFLAKE_PASSWORD}\""
+  elif [ -n "${SNOWFLAKE_PRIVATE_KEY_PATH}" ] || [ -n "${SNOWFLAKE_PRIVATE_KEY}" ]; then
+    # Key-based authentication
+    if [ -n "${SNOWFLAKE_PRIVATE_KEY_PATH}" ]; then
+      auth_command+=" --private-key-path \"${SNOWFLAKE_PRIVATE_KEY_PATH}\""
+    else
+      auth_command+=" --private-key \"${SNOWFLAKE_PRIVATE_KEY}\""
+    fi
+
+    # Add passphrase if provided
+    if [ -n "${SNOWFLAKE_PRIVATE_KEY_PASSPHRASE}" ]; then
+      auth_command+=" --private-key-passphrase \"${SNOWFLAKE_PRIVATE_KEY_PASSPHRASE}\""
+    fi
+  else
+    echo "Error: No authentication method provided for Snowflake. Please provide either password, private key path, or private key content."
+    exit 1
+  fi
+
+  # Add optional role and warehouse if provided
+  [ -n "${SNOWFLAKE_ROLE}" ] && auth_command+=" --role \"${SNOWFLAKE_ROLE}\""
+  [ -n "${SNOWFLAKE_WAREHOUSE}" ] && auth_command+=" --warehouse \"${SNOWFLAKE_WAREHOUSE}\""
+
+  eval $auth_command
   check_exit_status $? ""
 fi
 
@@ -121,6 +147,28 @@ if [ -n "${AWS_ACCESS_KEY_ID}" ]; then
   check_exit_status $? ""
 fi
 
+if [ -n "${BIGQUERY_PROJECT_ID}" ] || [ -n "${BIGQUERY_CREDENTIALS_JSON_PATH}" ]; then
+  echo "bigquery provider used: running 'sdf auth login bigquery'"
+
+  # Base command
+  auth_command="sdf auth login bigquery"
+
+  if [ -n "${BIGQUERY_CREDENTIALS_JSON_PATH}" ]; then
+    # Authentication method 2: Using credentials JSON file
+    auth_command+=" --json-path \"${BIGQUERY_CREDENTIALS_JSON_PATH}\""
+  else
+    # Authentication method 1: Using credentials directly
+    if [ -z "${BIGQUERY_PROJECT_ID}" ] || [ -z "${BIGQUERY_CLIENT_EMAIL}" ] || [ -z "${BIGQUERY_PRIVATE_KEY}" ]; then
+      echo "Error: For BigQuery individual credentials authentication, all of these are required: project_id, client_email, and private_key"
+      exit 1
+    fi
+    auth_command+=" --project-id \"${BIGQUERY_PROJECT_ID}\" --client-email \"${BIGQUERY_CLIENT_EMAIL}\" --private-key \"${BIGQUERY_PRIVATE_KEY}\""
+  fi
+
+  eval $auth_command
+  check_exit_status $? ""
+fi
+
 # run and save outputs
 echo "running command: $input_command"
 log=$($input_command 2>&1)

workspace/dbt_init_challenge/dbt_project.yml

@@ -22,6 +22,9 @@ clean-targets: # directories to be removed by `dbt clean`
   - "target"
   - "dbt_packages"
 
+vars:
+  project_name: "dbt_hol"
+
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models