From 2d3cf31fcf74dfa4e6a09ff4b242fe0112f4a90b Mon Sep 17 00:00:00 2001 From: xuliangs Date: Tue, 2 Apr 2024 22:45:21 +0000 Subject: [PATCH 1/5] Test aws credentials used in redshift compile --- .github/workflows/aws.yml | 34 ++++++++++++++++ action.yml | 19 ++++++++- workspace/tpch_redshift/queries/q1.sql | 21 ++++++++++ workspace/tpch_redshift/queries/q10.sql | 31 ++++++++++++++ workspace/tpch_redshift/queries/q11.sql | 28 +++++++++++++ workspace/tpch_redshift/queries/q12.sql | 30 ++++++++++++++ workspace/tpch_redshift/queries/q13.sql | 20 +++++++++ workspace/tpch_redshift/queries/q14.sql | 13 ++++++ workspace/tpch_redshift/queries/q15.sql | 32 +++++++++++++++ workspace/tpch_redshift/queries/q16.sql | 30 ++++++++++++++ workspace/tpch_redshift/queries/q17.sql | 17 ++++++++ workspace/tpch_redshift/queries/q18.sql | 33 +++++++++++++++ workspace/tpch_redshift/queries/q19.sql | 35 ++++++++++++++++ workspace/tpch_redshift/queries/q2.sql | 43 ++++++++++++++++++++ workspace/tpch_redshift/queries/q20.sql | 37 +++++++++++++++++ workspace/tpch_redshift/queries/q21.sql | 39 ++++++++++++++++++ workspace/tpch_redshift/queries/q22.sql | 37 +++++++++++++++++ workspace/tpch_redshift/queries/q3.sql | 22 ++++++++++ workspace/tpch_redshift/queries/q4.sql | 21 ++++++++++ workspace/tpch_redshift/queries/q5.sql | 24 +++++++++++ workspace/tpch_redshift/queries/q6.sql | 9 ++++ workspace/tpch_redshift/queries/q7.sql | 39 ++++++++++++++++++ workspace/tpch_redshift/queries/q8.sql | 37 +++++++++++++++++ workspace/tpch_redshift/queries/q9.sql | 32 +++++++++++++++ workspace/tpch_redshift/queries/revenue0.sql | 11 +++++ workspace/tpch_redshift/workspace.sdf.yml | 14 +++++++ 26 files changed, 706 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/aws.yml create mode 100644 workspace/tpch_redshift/queries/q1.sql create mode 100644 workspace/tpch_redshift/queries/q10.sql create mode 100644 workspace/tpch_redshift/queries/q11.sql create mode 100644 workspace/tpch_redshift/queries/q12.sql create mode 100644 workspace/tpch_redshift/queries/q13.sql create mode 100644 workspace/tpch_redshift/queries/q14.sql create mode 100644 workspace/tpch_redshift/queries/q15.sql create mode 100644 workspace/tpch_redshift/queries/q16.sql create mode 100644 workspace/tpch_redshift/queries/q17.sql create mode 100644 workspace/tpch_redshift/queries/q18.sql create mode 100644 workspace/tpch_redshift/queries/q19.sql create mode 100644 workspace/tpch_redshift/queries/q2.sql create mode 100644 workspace/tpch_redshift/queries/q20.sql create mode 100644 workspace/tpch_redshift/queries/q21.sql create mode 100644 workspace/tpch_redshift/queries/q22.sql create mode 100644 workspace/tpch_redshift/queries/q3.sql create mode 100644 workspace/tpch_redshift/queries/q4.sql create mode 100644 workspace/tpch_redshift/queries/q5.sql create mode 100644 workspace/tpch_redshift/queries/q6.sql create mode 100644 workspace/tpch_redshift/queries/q7.sql create mode 100644 workspace/tpch_redshift/queries/q8.sql create mode 100644 workspace/tpch_redshift/queries/q9.sql create mode 100644 workspace/tpch_redshift/queries/revenue0.sql create mode 100644 workspace/tpch_redshift/workspace.sdf.yml diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml new file mode 100644 index 0000000..a09a903 --- /dev/null +++ b/.github/workflows/aws.yml @@ -0,0 +1,34 @@ +name: Run Compile on branch + +on: + push: + branches: + - harry/aws_creds + pull_request: + +jobs: + tpch_redshift: + runs-on: ubuntu-latest + name: Run sdf compile + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: sdf compile step + uses: ./ # Uses an action in the root directory + id: sdf + with: + command: 'sdf compile --show all' + workspace_dir: 'workspace/tpch_redshift' + + aws_region: 'us-west-2' + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # Use the output from the `sdf` step + - name: Display the sdf output + run: | + echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + diff --git a/action.yml b/action.yml index dea521b..6f36e12 100644 --- a/action.yml +++ b/action.yml @@ -12,10 +12,10 @@ inputs: required: true access_key: description: "The access key" - required: true + required: false secret_key: description: "The secret key" - required: true + required: false is_dbt: description: "Whether the workspace is dbt based or not" required: true @@ -41,6 +41,17 @@ inputs: description: "The snowflake warehouse" required: false + # required if redshift table provider is used + aws_region: + description: "The aws region" + required: false + aws_access_key_id: + description: "The aws access key id" + required: false + aws_secret_access_key: + description: "The aws secret access key" + required: false + outputs: result: description: "The result (pass or failed)" @@ -63,3 +74,7 @@ runs: SNOWFLAKE_ROLE: ${{ inputs.snowflake_role }} SNOWFLAKE_WAREHOUSE: ${{ inputs.snowflake_warehouse }} DBT_TARGET: ${{ inputs.dbt_target }} + + AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }} + AWS_DEFAULT_REGION: ${{ inputs.aws_region }} diff --git a/workspace/tpch_redshift/queries/q1.sql b/workspace/tpch_redshift/queries/q1.sql new file mode 100644 index 0000000..a0fcf15 --- /dev/null +++ b/workspace/tpch_redshift/queries/q1.sql @@ -0,0 +1,21 @@ +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= date '1998-09-02' +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q10.sql b/workspace/tpch_redshift/queries/q10.sql new file mode 100644 index 0000000..dc4d01c --- /dev/null +++ b/workspace/tpch_redshift/queries/q10.sql @@ -0,0 +1,31 @@ +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= date '1993-10-01' + and o_orderdate < date '1994-01-01' + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q11.sql b/workspace/tpch_redshift/queries/q11.sql new file mode 100644 index 0000000..a294c9b --- /dev/null +++ b/workspace/tpch_redshift/queries/q11.sql @@ -0,0 +1,28 @@ +select + ps_partkey, + sum(ps_supplycost * ps_availqty) as value +from + partsupp, + supplier, + nation +where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' +group by + ps_partkey +having + sum(ps_supplycost * ps_availqty) > ( + select + sum(ps_supplycost * ps_availqty) * 0.0001 + from + partsupp, + supplier, + nation + where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'GERMANY' + ) +order by + value desc; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q12.sql b/workspace/tpch_redshift/queries/q12.sql new file mode 100644 index 0000000..f8e6d96 --- /dev/null +++ b/workspace/tpch_redshift/queries/q12.sql @@ -0,0 +1,30 @@ +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + lineitem + join + orders + on + l_orderkey = o_orderkey +where + l_shipmode in ('MAIL', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= date '1994-01-01' + and l_receiptdate < date '1995-01-01' +group by + l_shipmode +order by + l_shipmode; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q13.sql b/workspace/tpch_redshift/queries/q13.sql new file mode 100644 index 0000000..4bfe8c3 --- /dev/null +++ b/workspace/tpch_redshift/queries/q13.sql @@ -0,0 +1,20 @@ +select + c_count, + count(*) as custdist +from + ( + select + c_custkey, + count(o_orderkey) + from + customer left outer join orders on + c_custkey = o_custkey + and o_comment not like '%special%requests%' + group by + c_custkey + ) as c_orders (c_custkey, c_count) +group by + c_count +order by + custdist desc, + c_count desc; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q14.sql b/workspace/tpch_redshift/queries/q14.sql new file mode 100644 index 0000000..d8ef6af --- /dev/null +++ b/workspace/tpch_redshift/queries/q14.sql @@ -0,0 +1,13 @@ +select + 100.00 * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= date '1995-09-01' + and l_shipdate < date '1995-10-01'; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q15.sql b/workspace/tpch_redshift/queries/q15.sql new file mode 100644 index 0000000..bf1234b --- /dev/null +++ b/workspace/tpch_redshift/queries/q15.sql @@ -0,0 +1,32 @@ +-- create table revenue0 as +-- select +-- l_suppkey as supplier_no, +-- sum(l_extendedprice * (1 - l_discount)) as total_revenue +-- from +-- lineitem +-- where +-- l_shipdate >= date '1996-01-01' +-- and l_shipdate < date '1996-01-01' + interval '3' month +-- group by +-- l_suppkey; +select + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +from + supplier, + revenue0 +where + s_suppkey = supplier_no + and total_revenue = ( + select + max(total_revenue) + from + revenue0 + ) +order by + s_suppkey; + +--drop view revenue0; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q16.sql b/workspace/tpch_redshift/queries/q16.sql new file mode 100644 index 0000000..36b7c07 --- /dev/null +++ b/workspace/tpch_redshift/queries/q16.sql @@ -0,0 +1,30 @@ +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#45' + and p_type not like 'MEDIUM POLISHED%' + and p_size in (49, 14, 23, 45, 19, 3, 36, 9) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' +) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q17.sql b/workspace/tpch_redshift/queries/q17.sql new file mode 100644 index 0000000..1e65550 --- /dev/null +++ b/workspace/tpch_redshift/queries/q17.sql @@ -0,0 +1,17 @@ +select + sum(l_extendedprice) / 7.0 as avg_yearly +from + lineitem, + part +where + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container = 'MED BOX' + and l_quantity < ( + select + 0.2 * avg(l_quantity) + from + lineitem + where + l_partkey = p_partkey +); \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q18.sql b/workspace/tpch_redshift/queries/q18.sql new file mode 100644 index 0000000..edf8703 --- /dev/null +++ b/workspace/tpch_redshift/queries/q18.sql @@ -0,0 +1,33 @@ +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey + having + sum(l_quantity) > 300 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q19.sql b/workspace/tpch_redshift/queries/q19.sql new file mode 100644 index 0000000..56668e7 --- /dev/null +++ b/workspace/tpch_redshift/queries/q19.sql @@ -0,0 +1,35 @@ +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#12' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 1 and l_quantity <= 1 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 10 and l_quantity <= 10 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#34' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 20 and l_quantity <= 20 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ); \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q2.sql b/workspace/tpch_redshift/queries/q2.sql new file mode 100644 index 0000000..f66af21 --- /dev/null +++ b/workspace/tpch_redshift/queries/q2.sql @@ -0,0 +1,43 @@ +select + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +from + part, + supplier, + partsupp, + nation, + region +where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and p_size = 15 + and p_type like '%BRASS' + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' + and ps_supplycost = ( + select + min(ps_supplycost) + from + partsupp, + supplier, + nation, + region + where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'EUROPE' +) +order by + s_acctbal desc, + n_name, + s_name, + p_partkey; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q20.sql b/workspace/tpch_redshift/queries/q20.sql new file mode 100644 index 0000000..dd61a7d --- /dev/null +++ b/workspace/tpch_redshift/queries/q20.sql @@ -0,0 +1,37 @@ +select + s_name, + s_address +from + supplier, + nation +where + s_suppkey in ( + select + ps_suppkey + from + partsupp + where + ps_partkey in ( + select + p_partkey + from + part + where + p_name like 'forest%' + ) + and ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + ) + ) + and s_nationkey = n_nationkey + and n_name = 'CANADA' +order by + s_name; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q21.sql b/workspace/tpch_redshift/queries/q21.sql new file mode 100644 index 0000000..9d2fe32 --- /dev/null +++ b/workspace/tpch_redshift/queries/q21.sql @@ -0,0 +1,39 @@ +select + s_name, + count(*) as numwait +from + supplier, + lineitem l1, + orders, + nation +where + s_suppkey = l1.l_suppkey + and o_orderkey = l1.l_orderkey + and o_orderstatus = 'F' + and l1.l_receiptdate > l1.l_commitdate + and exists ( + select + * + from + lineitem l2 + where + l2.l_orderkey = l1.l_orderkey + and l2.l_suppkey <> l1.l_suppkey + ) + and not exists ( + select + * + from + lineitem l3 + where + l3.l_orderkey = l1.l_orderkey + and l3.l_suppkey <> l1.l_suppkey + and l3.l_receiptdate > l3.l_commitdate + ) + and s_nationkey = n_nationkey + and n_name = 'SAUDI ARABIA' +group by + s_name +order by + numwait desc, + s_name; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q22.sql b/workspace/tpch_redshift/queries/q22.sql new file mode 100644 index 0000000..90aea6f --- /dev/null +++ b/workspace/tpch_redshift/queries/q22.sql @@ -0,0 +1,37 @@ +select + cntrycode, + count(*) as numcust, + sum(c_acctbal) as totacctbal +from + ( + select + substring(c_phone from 1 for 2) as cntrycode, + c_acctbal + from + customer + where + substring(c_phone from 1 for 2) in + ('13', '31', '23', '29', '30', '18', '17') + and c_acctbal > ( + select + avg(c_acctbal) + from + customer + where + c_acctbal > 0.00 + and substring(c_phone from 1 for 2) in + ('13', '31', '23', '29', '30', '18', '17') + ) + and not exists ( + select + * + from + orders + where + o_custkey = c_custkey + ) + ) as custsale +group by + cntrycode +order by + cntrycode; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q3.sql b/workspace/tpch_redshift/queries/q3.sql new file mode 100644 index 0000000..7dbc6d9 --- /dev/null +++ b/workspace/tpch_redshift/queries/q3.sql @@ -0,0 +1,22 @@ +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'BUILDING' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < date '1995-03-15' + and l_shipdate > date '1995-03-15' +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q4.sql b/workspace/tpch_redshift/queries/q4.sql new file mode 100644 index 0000000..74a620d --- /dev/null +++ b/workspace/tpch_redshift/queries/q4.sql @@ -0,0 +1,21 @@ +select + o_orderpriority, + count(*) as order_count +from + orders +where + o_orderdate >= '1993-07-01' + and o_orderdate < date '1993-07-01' + interval '3' month + and exists ( + select + * + from + lineitem + where + l_orderkey = o_orderkey + and l_commitdate < l_receiptdate + ) +group by + o_orderpriority +order by + o_orderpriority; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q5.sql b/workspace/tpch_redshift/queries/q5.sql new file mode 100644 index 0000000..5a336b2 --- /dev/null +++ b/workspace/tpch_redshift/queries/q5.sql @@ -0,0 +1,24 @@ +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1995-01-01' +group by + n_name +order by + revenue desc; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q6.sql b/workspace/tpch_redshift/queries/q6.sql new file mode 100644 index 0000000..5806f98 --- /dev/null +++ b/workspace/tpch_redshift/queries/q6.sql @@ -0,0 +1,9 @@ +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= date '1994-01-01' + and l_shipdate < date '1995-01-01' + and l_discount between 0.06 - 0.01 and 0.06 + 0.01 + and l_quantity < 24; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q7.sql b/workspace/tpch_redshift/queries/q7.sql new file mode 100644 index 0000000..512e5be --- /dev/null +++ b/workspace/tpch_redshift/queries/q7.sql @@ -0,0 +1,39 @@ +select + supp_nation, + cust_nation, + l_year, + sum(volume) as revenue +from + ( + select + n1.n_name as supp_nation, + n2.n_name as cust_nation, + extract(year from l_shipdate) as l_year, + l_extendedprice * (1 - l_discount) as volume + from + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + where + s_suppkey = l_suppkey + and o_orderkey = l_orderkey + and c_custkey = o_custkey + and s_nationkey = n1.n_nationkey + and c_nationkey = n2.n_nationkey + and ( + (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY') + or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE') + ) + and l_shipdate between date '1995-01-01' and date '1996-12-31' + ) as shipping +group by + supp_nation, + cust_nation, + l_year +order by + supp_nation, + cust_nation, + l_year; diff --git a/workspace/tpch_redshift/queries/q8.sql b/workspace/tpch_redshift/queries/q8.sql new file mode 100644 index 0000000..6ddb2a6 --- /dev/null +++ b/workspace/tpch_redshift/queries/q8.sql @@ -0,0 +1,37 @@ +select + o_year, + sum(case + when nation = 'BRAZIL' then volume + else 0 + end) / sum(volume) as mkt_share +from + ( + select + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) as volume, + n2.n_name as nation + from + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + where + p_partkey = l_partkey + and s_suppkey = l_suppkey + and l_orderkey = o_orderkey + and o_custkey = c_custkey + and c_nationkey = n1.n_nationkey + and n1.n_regionkey = r_regionkey + and r_name = 'AMERICA' + and s_nationkey = n2.n_nationkey + and o_orderdate between date '1995-01-01' and date '1996-12-31' + and p_type = 'ECONOMY ANODIZED STEEL' + ) as all_nations +group by + o_year +order by + o_year; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/q9.sql b/workspace/tpch_redshift/queries/q9.sql new file mode 100644 index 0000000..587bbc8 --- /dev/null +++ b/workspace/tpch_redshift/queries/q9.sql @@ -0,0 +1,32 @@ +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%green%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc; \ No newline at end of file diff --git a/workspace/tpch_redshift/queries/revenue0.sql b/workspace/tpch_redshift/queries/revenue0.sql new file mode 100644 index 0000000..6cb5942 --- /dev/null +++ b/workspace/tpch_redshift/queries/revenue0.sql @@ -0,0 +1,11 @@ +create table revenue0 as +select + l_suppkey as supplier_no, + sum(l_extendedprice * (1 - l_discount)) as total_revenue +from + lineitem +where + l_shipdate >= date '1996-01-01' + and l_shipdate < date '1996-01-01' + interval '3' month +group by + l_suppkey; \ No newline at end of file diff --git a/workspace/tpch_redshift/workspace.sdf.yml b/workspace/tpch_redshift/workspace.sdf.yml new file mode 100644 index 0000000..74fb3dd --- /dev/null +++ b/workspace/tpch_redshift/workspace.sdf.yml @@ -0,0 +1,14 @@ +workspace: + edition: "1.1" + name: "tpch" + default-catalog: sample_data_dev + default-schema: tpch + compute: "local" + includes: + - path: queries/ +--- +provider: + type: redshift + cluster-identifier: sdf-cli-tests + sources: + - sample_data_dev From 74116279c4946a7eb08b3deae6eb21c44f4d5343 Mon Sep 17 00:00:00 2001 From: xuliangs Date: Tue, 2 Apr 2024 22:47:31 +0000 Subject: [PATCH 2/5] update sdf version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3b17258..45477b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.12.2-bookworm -ARG SDF_VERSION=0.1.170 +ARG SDF_VERSION=0.1.184 # Install dependencies RUN apt-get update && apt-get install -y \ From ee26e64a5023ad69473c2107a911977b099ae2f8 Mon Sep 17 00:00:00 2001 From: xuliangs Date: Tue, 2 Apr 2024 22:56:46 +0000 Subject: [PATCH 3/5] update workflow file --- .github/workflows/aws.yml | 34 ---------------------------------- .github/workflows/main.yml | 30 ++++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/aws.yml diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml deleted file mode 100644 index a09a903..0000000 --- a/.github/workflows/aws.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Run Compile on branch - -on: - push: - branches: - - harry/aws_creds - pull_request: - -jobs: - tpch_redshift: - runs-on: ubuntu-latest - name: Run sdf compile - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: sdf compile step - uses: ./ # Uses an action in the root directory - id: sdf - with: - command: 'sdf compile --show all' - workspace_dir: 'workspace/tpch_redshift' - - aws_region: 'us-west-2' - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - # Use the output from the `sdf` step - - name: Display the sdf output - run: | - echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY - echo '```' >>$GITHUB_STEP_SUMMARY - diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5146180..4405dfc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,15 +1,12 @@ name: On Pull Request Run Compile on: - push: - branches: - - main pull_request: jobs: dbt_init_challenge_job: runs-on: ubuntu-latest - name: Run sdf compile + name: Run sdf push on a DBT + Snowflake workspace steps: - name: Checkout uses: actions/checkout@v4 @@ -38,3 +35,28 @@ jobs: echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY echo '```' >>$GITHUB_STEP_SUMMARY + tpch_redshift: + runs-on: ubuntu-latest + name: Run sdf compile on a Redshift workspace + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: sdf compile step + uses: ./ # Uses an action in the root directory + id: sdf + with: + command: 'sdf compile --show all' + workspace_dir: 'workspace/tpch_redshift' + + aws_region: 'us-west-2' + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + # Use the output from the `sdf` step + - name: Display the sdf output + run: | + echo "### SDF Run Logs 🪵" >> $GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + echo "${{ steps.sdf.outputs.log }}" >>$GITHUB_STEP_SUMMARY + echo '```' >>$GITHUB_STEP_SUMMARY + From 959532700002c7cfa63fd7dee00042234906636f Mon Sep 17 00:00:00 2001 From: xuliangs Date: Tue, 2 Apr 2024 23:03:21 +0000 Subject: [PATCH 4/5] nit update --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4405dfc..add3968 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: On Pull Request Run Compile +name: Test sdf-action on: pull_request: From 84a5731ca2b195fe699c7a013848a1cb5f37df9d Mon Sep 17 00:00:00 2001 From: xuliangs Date: Thu, 4 Apr 2024 16:24:02 +0000 Subject: [PATCH 5/5] use the latest version --- Dockerfile | 2 +- Dockerfile.normal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45477b4..2c8d7c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.12.2-bookworm -ARG SDF_VERSION=0.1.184 +ARG SDF_VERSION=0.1.192 # Install dependencies RUN apt-get update && apt-get install -y \ diff --git a/Dockerfile.normal b/Dockerfile.normal index d06940a..062b8f6 100644 --- a/Dockerfile.normal +++ b/Dockerfile.normal @@ -1,5 +1,5 @@ FROM python:3.12.2-bookworm -ARG SDF_VERSION=0.1.170 +ARG SDF_VERSION=0.1.192 # Install dependencies RUN apt-get update && apt-get install -y \