Merge pull request #2 from sdf-labs/harry/aws_creds

Harry/allow passing in aws credentials

.github/workflows/main.yml

@@ -1,15 +1,12 @@
-name: On Pull Request Run Compile
+name: Test sdf-action
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
 
 jobs:
   dbt_init_challenge_job:
     runs-on: ubuntu-latest
-    name: Run sdf compile
+    name: Run sdf push on a DBT + Snowflake workspace
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -38,3 +35,28 @@ jobs:
           echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY
           echo '```' >>$GITHUB_STEP_SUMMARY
 
+  tpch_redshift:
+    runs-on: ubuntu-latest
+    name: Run sdf compile on a Redshift workspace
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: sdf compile step
+        uses: ./ # Uses an action in the root directory
+        id: sdf
+        with:
+          command: 'sdf compile --show all'
+          workspace_dir: 'workspace/tpch_redshift'
+
+          aws_region: 'us-west-2'
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      # Use the output from the `sdf` step
+      - name: Display the sdf output
+        run: |
+          echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY
+          echo '```' >>$GITHUB_STEP_SUMMARY
+          echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY
+          echo '```' >>$GITHUB_STEP_SUMMARY
+

Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12.2-bookworm
-ARG SDF_VERSION=0.1.170
+ARG SDF_VERSION=0.1.192
 
 # Install dependencies
 RUN apt-get update && apt-get install -y \

Dockerfile.normal

@@ -1,5 +1,5 @@
 FROM python:3.12.2-bookworm
-ARG SDF_VERSION=0.1.170
+ARG SDF_VERSION=0.1.192
 
 # Install dependencies
 RUN apt-get update && apt-get install -y \

action.yml

@@ -12,10 +12,10 @@ inputs:
     required: true
   access_key:
     description: "The access key"
-    required: true
+    required: false
   secret_key:
     description: "The secret key"
-    required: true
+    required: false
   is_dbt:
     description: "Whether the workspace is dbt based or not"
     required: true
@@ -41,6 +41,17 @@ inputs:
     description: "The snowflake warehouse"
     required: false
 
+  # required if redshift table provider is used
+  aws_region:
+    description: "The aws region"
+    required: false
+  aws_access_key_id:
+    description: "The aws access key id"
+    required: false
+  aws_secret_access_key:
+    description: "The aws secret access key"
+    required: false
+
 outputs:
   result:
     description: "The result (pass or failed)"
@@ -63,3 +74,7 @@ runs:
     SNOWFLAKE_ROLE: ${{ inputs.snowflake_role }}
     SNOWFLAKE_WAREHOUSE: ${{ inputs.snowflake_warehouse }}
     DBT_TARGET: ${{ inputs.dbt_target }}
+
+    AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+    AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+    AWS_DEFAULT_REGION: ${{ inputs.aws_region }}

workspace/tpch_redshift/queries/q1.sql

@@ -0,0 +1,21 @@
+select
+    l_returnflag,
+    l_linestatus,
+    sum(l_quantity) as sum_qty,
+    sum(l_extendedprice) as sum_base_price,
+    sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,
+    sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,
+    avg(l_quantity) as avg_qty,
+    avg(l_extendedprice) as avg_price,
+    avg(l_discount) as avg_disc,
+    count(*) as count_order
+from
+    lineitem
+where
+        l_shipdate <= date '1998-09-02'
+group by
+    l_returnflag,
+    l_linestatus
+order by
+    l_returnflag,
+    l_linestatus;

workspace/tpch_redshift/queries/q10.sql

@@ -0,0 +1,31 @@
+select
+    c_custkey,
+    c_name,
+    sum(l_extendedprice * (1 - l_discount)) as revenue,
+    c_acctbal,
+    n_name,
+    c_address,
+    c_phone,
+    c_comment
+from
+    customer,
+    orders,
+    lineitem,
+    nation
+where
+    c_custkey = o_custkey
+    and l_orderkey = o_orderkey
+    and o_orderdate >= date '1993-10-01'
+    and o_orderdate < date '1994-01-01'
+    and l_returnflag = 'R'
+    and c_nationkey = n_nationkey
+group by
+    c_custkey,
+    c_name,
+    c_acctbal,
+    c_phone,
+    n_name,
+    c_address,
+    c_comment
+order by
+    revenue desc;

workspace/tpch_redshift/queries/q11.sql

@@ -0,0 +1,28 @@
+select
+    ps_partkey,
+    sum(ps_supplycost * ps_availqty) as value
+from
+    partsupp,
+    supplier,
+    nation
+where
+    ps_suppkey = s_suppkey
+    and s_nationkey = n_nationkey
+    and n_name = 'GERMANY'
+group by
+    ps_partkey
+having
+    sum(ps_supplycost * ps_availqty) > (
+        select
+            sum(ps_supplycost * ps_availqty) * 0.0001
+        from
+            partsupp,
+            supplier,
+            nation
+        where
+            ps_suppkey = s_suppkey
+            and s_nationkey = n_nationkey
+            and n_name = 'GERMANY'
+    )
+order by
+    value desc;

workspace/tpch_redshift/queries/q12.sql

@@ -0,0 +1,30 @@
+select
+    l_shipmode,
+    sum(case
+            when o_orderpriority = '1-URGENT'
+                or o_orderpriority = '2-HIGH'
+                then 1
+            else 0
+        end) as high_line_count,
+    sum(case
+            when o_orderpriority <> '1-URGENT'
+                and o_orderpriority <> '2-HIGH'
+                then 1
+            else 0
+        end) as low_line_count
+from
+    lineitem
+        join
+    orders
+    on
+            l_orderkey = o_orderkey
+where
+        l_shipmode in ('MAIL', 'SHIP')
+  and l_commitdate < l_receiptdate
+  and l_shipdate < l_commitdate
+  and l_receiptdate >= date '1994-01-01'
+  and l_receiptdate < date '1995-01-01'
+group by
+    l_shipmode
+order by
+    l_shipmode;

workspace/tpch_redshift/queries/q13.sql

@@ -0,0 +1,20 @@
+select
+    c_count,
+    count(*) as custdist
+from
+    (
+        select
+            c_custkey,
+            count(o_orderkey)
+        from
+            customer left outer join orders on
+                        c_custkey = o_custkey
+                    and o_comment not like '%special%requests%'
+        group by
+            c_custkey
+    ) as c_orders (c_custkey, c_count)
+group by
+    c_count
+order by
+    custdist desc,
+    c_count desc;

workspace/tpch_redshift/queries/q14.sql

@@ -0,0 +1,13 @@
+select
+            100.00 * sum(case
+                             when p_type like 'PROMO%'
+                                 then l_extendedprice * (1 - l_discount)
+                             else 0
+            end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue
+from
+    lineitem,
+    part
+where
+        l_partkey = p_partkey
+  and l_shipdate >= date '1995-09-01'
+  and l_shipdate < date '1995-10-01';

workspace/tpch_redshift/queries/q15.sql

@@ -0,0 +1,32 @@
+-- create table revenue0 as
+-- 	select
+-- 		l_suppkey as supplier_no,
+-- 		sum(l_extendedprice * (1 - l_discount)) as total_revenue
+-- 	from
+-- 		lineitem
+-- 	where
+-- 		l_shipdate >= date '1996-01-01'
+-- 		and l_shipdate < date '1996-01-01' + interval '3' month
+-- 	group by
+-- 		l_suppkey;
+select
+	s_suppkey,
+	s_name,
+	s_address,
+	s_phone,
+	total_revenue
+from
+	supplier,
+	revenue0
+where
+	s_suppkey = supplier_no
+	and total_revenue = (
+		select
+			max(total_revenue)
+		from
+			revenue0
+	)
+order by
+	s_suppkey;
+
+--drop view revenue0;

workspace/tpch_redshift/queries/q16.sql

@@ -0,0 +1,30 @@
+select
+    p_brand,
+    p_type,
+    p_size,
+    count(distinct ps_suppkey) as supplier_cnt
+from
+    partsupp,
+    part
+where
+        p_partkey = ps_partkey
+  and p_brand <> 'Brand#45'
+  and p_type not like 'MEDIUM POLISHED%'
+  and p_size in (49, 14, 23, 45, 19, 3, 36, 9)
+  and ps_suppkey not in (
+    select
+        s_suppkey
+    from
+        supplier
+    where
+            s_comment like '%Customer%Complaints%'
+)
+group by
+    p_brand,
+    p_type,
+    p_size
+order by
+    supplier_cnt desc,
+    p_brand,
+    p_type,
+    p_size;

workspace/tpch_redshift/queries/q17.sql

@@ -0,0 +1,17 @@
+select
+        sum(l_extendedprice) / 7.0 as avg_yearly
+from
+    lineitem,
+    part
+where
+        p_partkey = l_partkey
+  and p_brand = 'Brand#23'
+  and p_container = 'MED BOX'
+  and l_quantity < (
+    select
+            0.2 * avg(l_quantity)
+    from
+        lineitem
+    where
+            l_partkey = p_partkey
+);

workspace/tpch_redshift/queries/q18.sql

@@ -0,0 +1,33 @@
+select
+    c_name,
+    c_custkey,
+    o_orderkey,
+    o_orderdate,
+    o_totalprice,
+    sum(l_quantity)
+from
+    customer,
+    orders,
+    lineitem
+where
+    o_orderkey in (
+        select
+            l_orderkey
+        from
+            lineitem
+        group by
+            l_orderkey
+        having
+            sum(l_quantity) > 300
+    )
+    and c_custkey = o_custkey
+    and o_orderkey = l_orderkey
+group by
+    c_name,
+    c_custkey,
+    o_orderkey,
+    o_orderdate,
+    o_totalprice
+order by
+    o_totalprice desc,
+    o_orderdate;