diff --git a/airflow/dags/mattermost_dags/extract/push_proxy_new.py b/airflow/dags/mattermost_dags/extract/push_proxy_new.py index 3a634258e..19fdf7971 100644 --- a/airflow/dags/mattermost_dags/extract/push_proxy_new.py +++ b/airflow/dags/mattermost_dags/extract/push_proxy_new.py @@ -82,7 +82,33 @@ def push_proxy_loader(): ], ) - extract_hpns_us_logs >> extract_tpns_logs + extract_hpns_eu_logs = KubernetesPodOperator( + **pod_defaults, + image=MATTERMOST_DATAWAREHOUSE_IMAGE, # Uses latest build from master + task_id="task-push-proxy-eu", + name=f"push-proxy-eu", + secrets=[ + SNOWFLAKE_LOAD_USER, + SNOWFLAKE_LOAD_PASSWORD, + SNOWFLAKE_ACCOUNT, + SNOWFLAKE_LOAD_DATABASE, + SNOWFLAKE_LOAD_WAREHOUSE, + SNOWFLAKE_LOAD_ROLE, + ], + env_vars={}, + arguments=[ + "push_proxy PUSH_PROXY_LOGS_EU_NEW LOGS_EU_NEW " + " --prefix /AWSLogs/{{ var.value.push_proxy_aws_account_id }}/elasticloadbalancing/{{ var.value.push_proxy_aws_region }}" + " -s {{ var.value.push_proxy_target_schema }}" + " -a ${SNOWFLAKE_ACCOUNT}" + " -d ${SNOWFLAKE_LOAD_DATABASE}" + " -w ${SNOWFLAKE_LOAD_WAREHOUSE}" + " -r ${SNOWFLAKE_LOAD_ROLE}" + " -u ${SNOWFLAKE_LOAD_USER}" + " -p ${SNOWFLAKE_LOAD_PASSWORD}" + ], + ) + extract_hpns_us_logs >> extract_tpns_logs >> extract_hpns_eu_logs dag = push_proxy_loader()