From 3631af44e6db0702e7797ca6ed994de451204764 Mon Sep 17 00:00:00 2001 From: Niall Turbitt Date: Mon, 11 Jul 2022 09:01:47 +0100 Subject: [PATCH] swap use of pool_id for original cluster spec --- conf/deployment.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/conf/deployment.yml b/conf/deployment.yml index e3a9e40..a04faf5 100644 --- a/conf/deployment.yml +++ b/conf/deployment.yml @@ -1,14 +1,17 @@ custom: # Cluster configs for each environment - # For demo purposes we will use a warm pool to acquire instances from default-cluster-spec: &default-cluster-spec spark_version: '11.0.x-cpu-ml-scala2.12' -# node_type_id: 'i3.xlarge' # node_type_id and driver_node_type_id cannot be supplied when an instance 'pool ID is provided -# driver_node_type_id: 'i3.xlarge' + node_type_id: 'i3.xlarge' + driver_node_type_id: 'i3.xlarge' num_workers: 1 - driver_instance_pool_id: '0617-151415-bells2-pool-hh7h6tjm' - instance_pool_id: '0617-151415-bells2-pool-hh7h6tjm' + # To reduce start up time for each job, it is advisable to use a cluster pool. To do so involves supplying the following + # two fields with a pool_id to acquire both the driver and instances from. + # If driver_instance_pool_id and instance_pool_id are set, both node_type_id and driver_node_type_id CANNOT be supplied. + # As such, if providing a pool_id for driver and worker instances, please ensure that node_type_id and driver_node_type_id are not present +# driver_instance_pool_id: '0617-151415-bells2-pool-hh7h6tjm' +# instance_pool_id: '0617-151415-bells2-pool-hh7h6tjm' dev-cluster-config: &dev-cluster-config new_cluster: