update spark properties for full utilization of node (#2276)

Azure · Feb 6, 2024 · 254dd2b · 254dd2b
1 parent 2d66471
commit 254dd2b
Show file tree

Hide file tree

Showing 19 changed files with 79 additions and 80 deletions.
diff --git a/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_drift/data_drift_compute_metrics/spec.yaml
@@ -40,10 +40,10 @@ outputs:
     mode: direct
     type: mltable
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_compute_metrics/spec.yaml
@@ -38,10 +38,10 @@ code: ../../src/
 entry:
   file: ./data_quality_compute_metrics/run.py
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_metrics_joiner/spec.yaml
@@ -27,10 +27,10 @@ code: ../../src/
 entry:
   file: ./data_quality_metrics_joiner/run.py
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml b/assets/model_monitoring/components/data_quality/data_quality_statistics/spec.yaml
@@ -26,10 +26,10 @@ code: ../../src/
 entry:
   file: ./data_quality_statistics/run.py
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/.../components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml b/.../components/feature_attribution_drift/feature_attribution_drift_compute_metrics/spec.yaml
@@ -25,12 +25,12 @@ outputs:
     mode: direct
 
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
-  spark.dynamicAllocation.enabled: False
+  spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1
   spark.dynamicAllocation.maxExecutors: 4
   spark.synapse.library.python.env: |

diff --git a/...odel_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml b/...odel_monitoring/components/feature_attribution_drift/feature_importance_metrics/spec.yaml
@@ -33,12 +33,12 @@ outputs:
     mode: direct
 
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
-  spark.dynamicAllocation.enabled: False
+  spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1
   spark.dynamicAllocation.maxExecutors: 4
   spark.synapse.library.python.env: |

diff --git a/...el_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml b/...el_monitoring/components/generation_safety_quality/annotation_compute_histogram/spec.yaml
@@ -86,10 +86,10 @@ outputs:
     mode: direct
 
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/...odel_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml b/...odel_monitoring/components/generation_safety_quality/annotation_compute_metrics/spec.yaml
@@ -41,10 +41,10 @@ outputs:
     mode: direct
 
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml b/assets/model_monitoring/components/model_monitor/model_data_collector_preprocessor/spec.yaml
@@ -27,11 +27,10 @@ outputs:
     type: mltable
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
-  spark.executor.instances: 1
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1
   spark.dynamicAllocation.maxExecutors: 4

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_compute_histogram/spec.yaml
@@ -29,10 +29,10 @@ outputs:
     mode: direct
     type: mltable
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/...del_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml b/...del_monitoring/components/model_monitor/model_monitor_compute_histogram_buckets/spec.yaml
@@ -30,10 +30,10 @@ outputs:
     mode: direct
     type: mltable
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_create_manifest/spec.yaml
@@ -56,10 +56,10 @@ outputs:
     type: uri_folder
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_data_joiner/spec.yaml
@@ -27,10 +27,10 @@ outputs:
     type: mltable
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/...el_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml b/...el_monitoring/components/model_monitor/model_monitor_evaluate_metrics_threshold/spec.yaml
@@ -4,7 +4,7 @@ type: spark
 name: model_monitor_evaluate_metrics_threshold
 display_name: Model Monitor - Evaluate Metrics Threshold
 description: Evaluate signal metrics against the threshold provided in the monitoring signal.
-version: 0.3.10
+version: 0.3.11
 is_deterministic: true
 
 code: ../../src
@@ -21,10 +21,10 @@ inputs:
     type: string
     optional: true
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_feature_selector/spec.yaml
@@ -4,7 +4,7 @@ type: spark
 name: model_monitor_feature_selector
 display_name: Model Monitor - Feature Selector
 description: Selects features to compute signal metrics on.
-version: 0.3.5
+version: 0.3.6
 is_deterministic: true
 
 code: ../../src
@@ -32,10 +32,10 @@ outputs:
     type: mltable
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_metric_outputter/spec.yaml
@@ -4,7 +4,7 @@ type: spark
 name: model_monitor_metric_outputter
 display_name: Model Monitor - Metric Outputter
 description: Output the computed model monitor metrics.
-version: 0.3.14
+version: 0.3.15
 is_deterministic: true
 
 code: ../../src/
@@ -32,10 +32,10 @@ outputs:
     type: uri_folder
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml b/assets/model_monitoring/components/model_monitor/model_monitor_output_metrics/spec.yaml
@@ -4,7 +4,7 @@ type: spark
 name: model_monitor_output_metrics
 display_name: Model Monitor - Output Metrics
 description: Output the computed model monitor metrics to the default datastore.
-version: 0.3.9
+version: 0.3.10
 is_deterministic: true
 
 code: ../../src/
@@ -40,10 +40,10 @@ outputs:
     type: uri_folder
     mode: direct
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/...model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml b/...model_monitoring/components/model_performance/model_performance_compute_metrics/spec.yaml
@@ -52,10 +52,10 @@ outputs:
     mode: direct
 
 conf:
-  spark.driver.cores: 1
-  spark.driver.memory: 2g
-  spark.executor.cores: 2
-  spark.executor.memory: 2g
+  spark.driver.cores: 4
+  spark.driver.memory: 28g
+  spark.executor.cores: 4
+  spark.executor.memory: 28g
   spark.executor.instances: 1
   spark.dynamicAllocation.enabled: True
   spark.dynamicAllocation.minExecutors: 1

diff --git a/assets/model_monitoring/components/tests/e2e/test_model_data_collector_preprocessor_e2e.py b/assets/model_monitoring/components/tests/e2e/test_model_data_collector_preprocessor_e2e.py
@@ -31,7 +31,7 @@ def _mdc_preprocessor_e2e():
 
         mdc_preprocessor_output.identity = AmlTokenConfiguration()
         mdc_preprocessor_output.resources = {
-            'instance_type': 'Standard_E8S_V3',
+            'instance_type': 'Standard_E4S_V3',
             'runtime_version': '3.3',
         }