MM-56793: add EU log ingestion (#1464)

* MM-56793: add EU log ingestion * MM-56793: fix task name * MM-56793: fix task name
mattermost · Mar 20, 2024 · f2abdeb · f2abdeb
1 parent 36b21d0
commit f2abdeb
Showing 1 changed file with 27 additions and 1 deletion.
diff --git a/airflow/dags/mattermost_dags/extract/push_proxy_new.py b/airflow/dags/mattermost_dags/extract/push_proxy_new.py
@@ -82,7 +82,33 @@ def push_proxy_loader():
         ],
     )
 
-    extract_hpns_us_logs >> extract_tpns_logs
+    extract_hpns_eu_logs = KubernetesPodOperator(
+        **pod_defaults,
+        image=MATTERMOST_DATAWAREHOUSE_IMAGE,  # Uses latest build from master
+        task_id="task-push-proxy-eu",
+        name=f"push-proxy-eu",
+        secrets=[
+            SNOWFLAKE_LOAD_USER,
+            SNOWFLAKE_LOAD_PASSWORD,
+            SNOWFLAKE_ACCOUNT,
+            SNOWFLAKE_LOAD_DATABASE,
+            SNOWFLAKE_LOAD_WAREHOUSE,
+            SNOWFLAKE_LOAD_ROLE,
+        ],
+        env_vars={},
+        arguments=[
+            "push_proxy PUSH_PROXY_LOGS_EU_NEW LOGS_EU_NEW "
+            " --prefix /AWSLogs/{{ var.value.push_proxy_aws_account_id }}/elasticloadbalancing/{{ var.value.push_proxy_aws_region }}"
+            " -s {{ var.value.push_proxy_target_schema }}"
+            " -a ${SNOWFLAKE_ACCOUNT}"
+            " -d ${SNOWFLAKE_LOAD_DATABASE}"
+            " -w ${SNOWFLAKE_LOAD_WAREHOUSE}"
+            " -r ${SNOWFLAKE_LOAD_ROLE}"
+            " -u ${SNOWFLAKE_LOAD_USER}"
+            " -p ${SNOWFLAKE_LOAD_PASSWORD}"
+        ],
+    )
+    extract_hpns_us_logs >> extract_tpns_logs >> extract_hpns_eu_logs
 
 
 dag = push_proxy_loader()