From 0de7592b89bbb3549cd3c0f27fbee0b9d5b6002c Mon Sep 17 00:00:00 2001 From: Jeyaprakash-NK Date: Tue, 13 Aug 2024 12:36:07 +0530 Subject: [PATCH 1/9] Fix - Properties value truncated with multi colon --- src/jobs/labelProperties.tsx | 4 ++-- src/runtime/createRunTime.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jobs/labelProperties.tsx b/src/jobs/labelProperties.tsx index 4ece4760..f35f5b66 100644 --- a/src/jobs/labelProperties.tsx +++ b/src/jobs/labelProperties.tsx @@ -151,7 +151,7 @@ function LabelProperties({ if (data.split(':')[1] === '') { data = data + value; } else { - data = data.replace(data.split(':')[1], value); + data = data.replace(data.split(/:(.+)/)[1], value); } } } @@ -287,7 +287,7 @@ function LabelProperties({ } defaultValue={ labelSplit.length > 2 - ? labelSplit[1] + ':' + labelSplit[2] + ? label.split(/:(.+)/)[1] : labelSplit[1] } Label={`Value ${index + 1}`} diff --git a/src/runtime/createRunTime.tsx b/src/runtime/createRunTime.tsx index befce2cb..876a737b 100644 --- a/src/runtime/createRunTime.tsx +++ b/src/runtime/createRunTime.tsx @@ -926,7 +926,7 @@ function CreateRunTime({ propertyObject[key] = value; }); propertyDetailUpdated.forEach((label: string) => { - const labelSplit = label.split(':'); + const labelSplit = label.split(/:(.+)/); const key = labelSplit[0]; const value = labelSplit[1]; propertyObject[key] = value; From 19901f82af20883ad6507d921e42d76dff09b68b Mon Sep 17 00:00:00 2001 From: Jeyaprakash-NK Date: Mon, 19 Aug 2024 15:27:00 +0530 Subject: [PATCH 2/9] spark properties - l4 and nonl4 changes --- src/runtime/createRunTime.tsx | 93 +++++++++++++++++++++++++++------ src/runtime/sparkProperties.tsx | 14 ++--- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/src/runtime/createRunTime.tsx b/src/runtime/createRunTime.tsx index befce2cb..9030e725 100644 --- a/src/runtime/createRunTime.tsx +++ b/src/runtime/createRunTime.tsx @@ -194,6 +194,8 @@ function CreateRunTime({ [] ); const [sharedvpcSelected, setSharedvpcSelected] = useState(''); + const [gpuDetailChangeDone, setGpuDetailChangeDone] = useState(false); + useEffect(() => { checkConfig(setLoggedIn, setConfigError, setLoginError); const localstorageGetInformation = localStorage.getItem('loginState'); @@ -246,41 +248,95 @@ function CreateRunTime({ const [key, value] = item.split(':'); if (key === 'spark.dataproc.executor.resource.accelerator.type') { if (value === 'l4') { - resourceAllocationModify = resourceAllocationDetailUpdated + resourceAllocationModify = resourceAllocationModify .map((item: string) => { - if (item === 'spark.dataproc.executor.disk.size:400g') { + if (item.includes('spark.dataproc.executor.disk.size')) { // To remove the property if GPU checkbox is checked and 'spark.dataproc.executor.resource.accelerator.type:l4'. return null; + } else if (item === 'spark.executor.cores:12') { + return 'spark.executor.cores:4'; } return item; }) .filter((item): item is string => item !== null); // To filter out null values.' setResourceAllocationDetail(resourceAllocationModify); setResourceAllocationDetailUpdated(resourceAllocationModify); + + let gpuDetailModify = [...gpuDetailUpdated]; + resourceAllocationModify.forEach(item => { + const [key, value] = item.split(':'); + if (key === 'spark.executor.cores') { + const cores = Number(value); + const gpuValue = (1 / cores).toFixed(2); + gpuDetailModify = gpuDetailModify.map(gpuItem => { + const [gpuKey] = gpuItem.split(':'); + if (gpuKey === 'spark.task.resource.gpu.amount') { + return `spark.task.resource.gpu.amount:${gpuValue}`; + } + return gpuItem; + }); + } + }); + setGpuDetail(gpuDetailModify); + setGpuDetailUpdated(gpuDetailModify); + setGpuDetailChangeDone(true); } else { + resourceAllocationModify = resourceAllocationModify + .map((item: string) => { + if (item === 'spark.executor.cores:4') { + return 'spark.executor.cores:12'; + } + return item; + }) + .filter((item): item is string => item !== null); // To filter out null values. + setResourceAllocationDetail(resourceAllocationModify); + setResourceAllocationDetailUpdated(resourceAllocationModify); + if ( - !resourceAllocationDetailUpdated.includes( - 'spark.dataproc.executor.disk.size:400g' - ) + resourceAllocationModify.filter(property => + property.includes('spark.dataproc.executor.disk.size') + ).length === 0 ) { - // To add the spark.dataproc.executor.disk.size:400g at index 9. - resourceAllocationDetailUpdated.splice( + // To add the spark.dataproc.executor.disk.size:750g at index 9. + resourceAllocationModify.splice( 8, 0, - 'spark.dataproc.executor.disk.size:400g' + 'spark.dataproc.executor.disk.size:750g' ); - const updatedArray = [...resourceAllocationDetailUpdated]; + const updatedArray = [...resourceAllocationModify]; setResourceAllocationDetail(updatedArray); setResourceAllocationDetailUpdated(updatedArray); } + + let gpuDetailModify = [...gpuDetailUpdated]; + resourceAllocationModify.forEach(item => { + const [key, value] = item.split(':'); + if (key === 'spark.executor.cores') { + const cores = Number(value); + const gpuValue = (1 / cores).toFixed(2); + gpuDetailModify = gpuDetailModify.map(gpuItem => { + const [gpuKey] = gpuItem.split(':'); + if (gpuKey === 'spark.task.resource.gpu.amount') { + return `spark.task.resource.gpu.amount:${gpuValue}`; + } + return gpuItem; + }); + } + }); + setGpuDetail(gpuDetailModify); + setGpuDetailUpdated(gpuDetailModify); + setGpuDetailChangeDone(true); } } }); + setResourceAllocationDetail(resourceAllocationModify); + setResourceAllocationDetailUpdated(resourceAllocationModify); }; - useEffect(() => { - modifyResourceAllocation(); - }, [gpuDetailUpdated]); + if (!gpuDetailChangeDone) { + modifyResourceAllocation(); + } + }, [gpuDetailUpdated, gpuDetailChangeDone]); const displayUserInfo = async () => { await RunTimeSerive.displayUserInfoService(setUserInfo); @@ -402,9 +458,11 @@ function CreateRunTime({ if (gpuChecked || gpuDetailList.length > 0) { setGpuDetail(gpuDetailList); setGpuDetailUpdated(gpuDetailList); + setGpuDetailChangeDone(false); } else { setGpuDetail(['']); setGpuDetailUpdated(['']); + setGpuDetailChangeDone(false); } setPropertyDetail(prevPropertyDetail => { @@ -1077,6 +1135,7 @@ function CreateRunTime({ }); setGpuDetail(gpuDetailModify); setGpuDetailUpdated(gpuDetailModify); + setGpuDetailChangeDone(false); } else { let resourceAllocationModify = [...resourceAllocationDetailUpdated]; resourceAllocationModify = resourceAllocationModify.map( @@ -1100,9 +1159,9 @@ function CreateRunTime({ ); } if ( - !resourceAllocationModify.includes( - 'spark.dataproc.executor.disk.size:400g' - ) + resourceAllocationModify.filter(property => + property.includes('spark.dataproc.executor.disk.size') + ).length === 0 ) { // To add the spark.dataproc.executor.disk.size:400g at index 9 when GPU is unchecked resourceAllocationModify.splice( @@ -1116,6 +1175,7 @@ function CreateRunTime({ setExpandGpu(false); setGpuDetail(['']); setGpuDetailUpdated(['']); + setGpuDetailChangeDone(false); } }; @@ -1640,6 +1700,7 @@ function CreateRunTime({ sparkValueValidation={sparkValueValidation} setSparkValueValidation={setSparkValueValidation} sparkSection="resourceallocation" + setGpuDetailChangeDone={setGpuDetailChangeDone} /> )}
@@ -1684,6 +1745,7 @@ function CreateRunTime({ sparkValueValidation={sparkValueValidation} setSparkValueValidation={setSparkValueValidation} sparkSection="autoscaling" + setGpuDetailChangeDone={setGpuDetailChangeDone} /> )}
@@ -1737,6 +1799,7 @@ function CreateRunTime({ sparkValueValidation={sparkValueValidation} setSparkValueValidation={setSparkValueValidation} sparkSection="gpu" + setGpuDetailChangeDone={setGpuDetailChangeDone} /> )}
Others
diff --git a/src/runtime/sparkProperties.tsx b/src/runtime/sparkProperties.tsx index ecd794f2..7e635421 100644 --- a/src/runtime/sparkProperties.tsx +++ b/src/runtime/sparkProperties.tsx @@ -42,7 +42,8 @@ function SparkProperties({ setLabelDetailUpdated, sparkValueValidation, setSparkValueValidation, - sparkSection + sparkSection, + setGpuDetailChangeDone }: any) { /* labelDetail used to store the permanent label details when onblur @@ -70,6 +71,9 @@ function SparkProperties({ const handleEditLabelSwitch = () => { setLabelDetail(labelDetailUpdated); + if (sparkSection === 'gpu') { + setGpuDetailChangeDone(false); + } }; const updateErrorIndexes = (index: number, hasError: boolean) => { @@ -177,11 +181,9 @@ function SparkProperties({ value is split from labels Example:"client:dataproc_jupyter_plugin" */ - if (data.split(':')[1] === '') { - data = data + value; - } else { - data = data.replace(data.split(':')[1], value); - } + let sparkProperties = data.split(':'); + sparkProperties[1] = value.trim(); + data = sparkProperties[0] + ':' + sparkProperties[1]; } labelEdit[dataNumber] = data; }); From ba30a8d9c3a68c04c8c1e440b7b8e15390241916 Mon Sep 17 00:00:00 2001 From: Jeyaprakash-NK Date: Tue, 20 Aug 2024 12:16:19 +0530 Subject: [PATCH 3/9] gpu checkbox uncheck handled --- src/runtime/createRunTime.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/createRunTime.tsx b/src/runtime/createRunTime.tsx index 9030e725..21f252cd 100644 --- a/src/runtime/createRunTime.tsx +++ b/src/runtime/createRunTime.tsx @@ -1142,6 +1142,8 @@ function CreateRunTime({ (item: string) => { if (item === 'spark.dataproc.executor.disk.tier:premium') { return 'spark.dataproc.executor.disk.tier:standard'; + } else if (item.includes('spark.executor.cores')) { + return 'spark.executor.cores:4' } return item; } @@ -1169,7 +1171,17 @@ function CreateRunTime({ 0, 'spark.dataproc.executor.disk.size:400g' ); + } else { + resourceAllocationModify = resourceAllocationModify.map( + (item: string) => { + if (item.includes('spark.dataproc.executor.disk.size')) { + return 'spark.dataproc.executor.disk.size:400g'; + } + return item; + } + ); } + setResourceAllocationDetail(resourceAllocationModify); setResourceAllocationDetailUpdated(resourceAllocationModify); setExpandGpu(false); From f374b0b5e9a30a6f68f57d19a8648d9c58126ccd Mon Sep 17 00:00:00 2001 From: Jeyaprakash-NK Date: Tue, 20 Aug 2024 15:51:53 +0530 Subject: [PATCH 4/9] clone bug fix --- src/runtime/createRunTime.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/runtime/createRunTime.tsx b/src/runtime/createRunTime.tsx index 21f252cd..f6b27ded 100644 --- a/src/runtime/createRunTime.tsx +++ b/src/runtime/createRunTime.tsx @@ -333,7 +333,17 @@ function CreateRunTime({ setResourceAllocationDetailUpdated(resourceAllocationModify); }; useEffect(() => { - if (!gpuDetailChangeDone) { + if ( + !gpuDetailChangeDone && + (!selectedRuntimeClone || + selectedRuntimeClone.runtimeConfig.properties[ + 'spark.dataproc.executor.resource.accelerator.type' + ] === 'l4' || + gpuDetailUpdated.includes( + 'spark.dataproc.executor.resource.accelerator.type:l4' + ) || + resourceAllocationDetailUpdated.length === 9) + ) { modifyResourceAllocation(); } }, [gpuDetailUpdated, gpuDetailChangeDone]); @@ -1143,7 +1153,7 @@ function CreateRunTime({ if (item === 'spark.dataproc.executor.disk.tier:premium') { return 'spark.dataproc.executor.disk.tier:standard'; } else if (item.includes('spark.executor.cores')) { - return 'spark.executor.cores:4' + return 'spark.executor.cores:4'; } return item; } From 700c707d117dc0c40dcde9d6d88b4025dfd2fe6e Mon Sep 17 00:00:00 2001 From: saranyaloganathan23 Date: Thu, 29 Aug 2024 11:38:52 +0530 Subject: [PATCH 5/9] checking papermill file exists --- dataproc_jupyter_plugin/services/executor.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dataproc_jupyter_plugin/services/executor.py b/dataproc_jupyter_plugin/services/executor.py index 34b1bb52..8e0c69eb 100644 --- a/dataproc_jupyter_plugin/services/executor.py +++ b/dataproc_jupyter_plugin/services/executor.py @@ -17,6 +17,8 @@ import subprocess import uuid from datetime import datetime, timedelta +from google.cloud import storage +from google.api_core.exceptions import NotFound import aiohttp import pendulum @@ -88,12 +90,17 @@ async def get_bucket(self, runtime_env): async def check_file_exists(self, bucket, file_path): try: - cmd = f"gsutil ls gs://{bucket}/dataproc-notebooks/{file_path}" - await async_run_gsutil_subcommand(cmd) - return True - except subprocess.CalledProcessError as error: - self.log.exception(f"Error checking papermill file: {error.decode()}") - raise IOError(error.decode) + if not bucket: + raise ValueError("Bucket name cannot be empty") + bucket_name = storage.Client().bucket(bucket) + blob = bucket_name.blob(file_path) + if blob.exists(): + return True + else: + return False + except Exception as error: + self.log.exception(f"Error checking file: {error}") + raise IOError(f"Error checking file: {error}") async def upload_papermill_to_gcs(self, gcs_dag_bucket): env = Environment( From 8cdd57b19669cb87d8b9891a154adc168ccccd95 Mon Sep 17 00:00:00 2001 From: saranyaloganathan23 Date: Thu, 29 Aug 2024 13:17:05 +0530 Subject: [PATCH 6/9] adding dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dbf97a6a..f126ad39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,8 @@ dependencies = [ "pendulum>=3.0.0", "pydantic~=1.10.0", "bigframes~=0.22.0", - "aiohttp~=3.9.5" + "aiohttp~=3.9.5", + "google-cloud-storage~=2.18.2" ] dynamic = ["version", "description", "authors", "urls", "keywords"] From dc615106e1f9ebf950ed58dd587abc9ec60d66b5 Mon Sep 17 00:00:00 2001 From: saranyaloganathan23 Date: Thu, 29 Aug 2024 19:04:51 +0530 Subject: [PATCH 7/9] changing exception error msg --- dataproc_jupyter_plugin/services/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproc_jupyter_plugin/services/executor.py b/dataproc_jupyter_plugin/services/executor.py index 8e0c69eb..b1ff0138 100644 --- a/dataproc_jupyter_plugin/services/executor.py +++ b/dataproc_jupyter_plugin/services/executor.py @@ -100,7 +100,7 @@ async def check_file_exists(self, bucket, file_path): return False except Exception as error: self.log.exception(f"Error checking file: {error}") - raise IOError(f"Error checking file: {error}") + raise IOError(f"Error creating dag: {error}") async def upload_papermill_to_gcs(self, gcs_dag_bucket): env = Environment( From 1f5229bf93bf9f733dd073fabbadde2d6dfbb7d6 Mon Sep 17 00:00:00 2001 From: saranyaloganathan23 Date: Fri, 30 Aug 2024 10:15:55 +0530 Subject: [PATCH 8/9] if else changes --- dataproc_jupyter_plugin/services/executor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dataproc_jupyter_plugin/services/executor.py b/dataproc_jupyter_plugin/services/executor.py index b1ff0138..17a4c2ba 100644 --- a/dataproc_jupyter_plugin/services/executor.py +++ b/dataproc_jupyter_plugin/services/executor.py @@ -94,10 +94,7 @@ async def check_file_exists(self, bucket, file_path): raise ValueError("Bucket name cannot be empty") bucket_name = storage.Client().bucket(bucket) blob = bucket_name.blob(file_path) - if blob.exists(): - return True - else: - return False + return blob.exists() except Exception as error: self.log.exception(f"Error checking file: {error}") raise IOError(f"Error creating dag: {error}") From 78bb5d0971010d338ce6a6349ad8c691850288f8 Mon Sep 17 00:00:00 2001 From: saranyaloganathan23 Date: Thu, 5 Sep 2024 11:44:11 +0530 Subject: [PATCH 9/9] variable name change --- dataproc_jupyter_plugin/services/executor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataproc_jupyter_plugin/services/executor.py b/dataproc_jupyter_plugin/services/executor.py index 17a4c2ba..f8517702 100644 --- a/dataproc_jupyter_plugin/services/executor.py +++ b/dataproc_jupyter_plugin/services/executor.py @@ -88,12 +88,12 @@ async def get_bucket(self, runtime_env): self.log.exception(f"Error getting bucket name: {str(e)}") raise Exception(f"Error getting composer bucket: {str(e)}") - async def check_file_exists(self, bucket, file_path): + async def check_file_exists(self, bucket_name, file_path): try: - if not bucket: + if not bucket_name: raise ValueError("Bucket name cannot be empty") - bucket_name = storage.Client().bucket(bucket) - blob = bucket_name.blob(file_path) + bucket = storage.Client().bucket(bucket_name) + blob = bucket.blob(file_path) return blob.exists() except Exception as error: self.log.exception(f"Error checking file: {error}")