oui is daily operator

manifests/poc-data-cluster/prod/spark/iot-netid-counts-silver.yaml

@@ -7,7 +7,7 @@ data:
   query.sql: |
     SELECT 
       date, net_id, count(net_id) AS count
-    FROM TABLE_IOT_PACKETS GROUP BY net_d ORDER BY neti_d DESC
+    FROM TABLE_IOT_PACKETS GROUP BY net_id ORDER BY net_id DESC
 ---
 apiVersion: "sparkoperator.k8s.io/v1beta2"
 kind: SparkApplication

manifests/poc-data-cluster/prod/spark/iot-oui-counts-silver.yaml

@@ -7,77 +7,85 @@ data:
   query.sql: |
     SELECT 
       date, oui, count(oui) AS count
-    FROM TABLE_IOT_PACKETS GROUP BY oui ORDER BY oui DESC
+    FROM TABLE_IOT_PACKETS
+    GROUP BY oui
+    ORDER BY oui DESC
+    WHERE date_add(current_date(), -1);
 ---
 apiVersion: "sparkoperator.k8s.io/v1beta2"
-kind: SparkApplication
+kind: ScheduledSparkApplication
 metadata:
   name: iot-oui-counts-silver
   namespace: spark
 spec:
-  type: Scala
-  mode: cluster
-  image: "public.ecr.aws/k0m1p4t7/spark:v3.4.0-aws"
-  imagePullPolicy: Always
-  mainClass: Main
-  mainApplicationFile: "s3a://foundation-data-lake-requester-pays/jars/spark-streaming-sql-assembly-1.0.jar"
-  sparkVersion: "3.4.0"
-  restartPolicy:
-    type: OnFailure
-    onFailureRetries: 3
-    onFailureRetryInterval: 10
-    onSubmissionFailureRetries: 3
-    onSubmissionFailureRetryInterval: 10
-  sparkConf:
-    spark.databricks.delta.autoCompact.enabled: "true"
-  hadoopConf:
-    fs.s3a.aws.credentials.provider: com.amazonaws.auth.WebIdentityTokenCredentialsProvider
-  volumes:
-    - name: "tmp"
-      hostPath:
-        path: "/tmp"
-        type: Directory
-    - name: config-vol
-      configMap:
-        name: iot-oui-counts-silver-query
-        items:
-          - key: query.sql
-            path: query.sql
-  driver:
-    serviceAccount: spark-data-lake-access
-    cores: 1
-    coreLimit: "1200m"
-    memory: "512m"
-    nodeSelector:
-      node.kubernetes.io/instance-type: m5.large
-    envVars:
-      TABLE_IOT_PACKETS: s3a://foundation-data-lake-requester-pays/silver/iot-packets
-      PARTITION_BY: "date"
-      CHECKPOINT: s3a://foundation-data-lake-requester-pays/checkpoints/iot-oui-counts
-      OUTPUT: s3a://foundation-data-lake-requester-pays/silver/iot-oui-counts
-      QUERY_PATH: /app/query.sql
-    labels:
-      version: 3.4.0
-    volumeMounts:
-      - name: "test-volume"
-        mountPath: "/tmp"
-      - name: config-vol
-        mountPath: /app
-  executor:
-    serviceAccount: spark-data-lake-access
-    cores: 1
-    coreLimit: "1200m"
-    instances: 3
-    memory: "10G"
-    tolerations: # Schedule executor pods on spot instance group
-      - key: dedicated
-        operator: Equal
-        value: spark
-        effect: NoSchedule
-    nodeSelector:
-      nodegroup-type: spot
-    labels:
-      version: 3.4.0
-    volumeMounts:
+  schedule: "@daily"
+  concurrencyPolicy: Allow
+  successfulRunHistoryLimit: 1
+  failedRunHistoryLimit: 3
+  template:
+    type: Scala
+    mode: cluster
+    image: "public.ecr.aws/k0m1p4t7/spark:v3.4.0-aws"
+    imagePullPolicy: Always
+    mainClass: Main
+    mainApplicationFile: "s3a://foundation-data-lake-requester-pays/jars/spark-streaming-sql-assembly-1.0.jar"
+    sparkVersion: "3.4.0"
+    restartPolicy:
+      type: OnFailure
+      onFailureRetries: 3
+      onFailureRetryInterval: 10
+      onSubmissionFailureRetries: 3
+      onSubmissionFailureRetryInterval: 10
+    sparkConf:
+      spark.databricks.delta.autoCompact.enabled: "true"
+    hadoopConf:
+      fs.s3a.aws.credentials.provider: com.amazonaws.auth.WebIdentityTokenCredentialsProvider
+    volumes:
       - name: "tmp"
-        mountPath: "/tmp"
+        hostPath:
+          path: "/tmp"
+          type: Directory
+      - name: config-vol
+        configMap:
+          name: iot-oui-counts-silver-query
+          items:
+            - key: query.sql
+              path: query.sql
+    driver:
+      serviceAccount: spark-data-lake-access
+      cores: 1
+      coreLimit: "1200m"
+      memory: "512m"
+      nodeSelector:
+        node.kubernetes.io/instance-type: m5.large
+      envVars:
+        TABLE_IOT_PACKETS: s3a://foundation-data-lake-requester-pays/silver/iot-packets
+        PARTITION_BY: "date"
+        CHECKPOINT: s3a://foundation-data-lake-requester-pays/checkpoints/iot-oui-counts
+        OUTPUT: s3a://foundation-data-lake-requester-pays/silver/iot-oui-counts
+        QUERY_PATH: /app/query.sql
+      labels:
+        version: 3.4.0
+      volumeMounts:
+        - name: "test-volume"
+          mountPath: "/tmp"
+        - name: config-vol
+          mountPath: /app
+    executor:
+      serviceAccount: spark-data-lake-access
+      cores: 1
+      coreLimit: "1200m"
+      instances: 3
+      memory: "10G"
+      tolerations: # Schedule executor pods on spot instance group
+        - key: dedicated
+          operator: Equal
+          value: spark
+          effect: NoSchedule
+      nodeSelector:
+        nodegroup-type: spot
+      labels:
+        version: 3.4.0
+      volumeMounts:
+        - name: "tmp"
+          mountPath: "/tmp"