From ab45305ac66323135a0a79938d88e5ac127f6261 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 7 Feb 2025 14:50:20 -0800 Subject: [PATCH 1/4] AutoTuner should recommend Dataproc Spark performance enhancements Signed-off-by: Partho Sarthi --- .../nvidia/spark/rapids/tool/Platform.scala | 5 ++++ .../tool/tuning/ProfilingAutoTunerSuite.scala | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala index 9e285bf5a..2fea6aff4 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/Platform.scala @@ -590,6 +590,11 @@ class DataprocPlatform(gpuDevice: Option[GpuDevice], clusterProperties: Option[ClusterProperties]) extends Platform(gpuDevice, clusterProperties) { override val platformName: String = PlatformNames.DATAPROC override val defaultGpuDevice: GpuDevice = T4Gpu + override val recommendationsToInclude: Seq[(String, String)] = Seq( + "spark.dataproc.enhanced.optimizer.enabled" -> "true", + "spark.dataproc.enhanced.execution.enabled" -> "true" + ) + override def isPlatformCSP: Boolean = true override def maxGpusSupported: Int = 4 diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala index f1cd67f9c..855654822 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala @@ -2989,4 +2989,32 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. // scalastyle:on line.size.limit compareOutput(expectedResults, autoTunerOutput) } + + test("test AutoTuner sets Dataproc Spark performance enhancements") { + // mock the properties loaded from eventLog + val logEventsProps: mutable.Map[String, String] = + mutable.LinkedHashMap[String, String]( + "spark.executor.cores" -> "16", + "spark.executor.instances" -> "1", + "spark.executor.memory" -> "80g", + "spark.executor.resource.gpu.amount" -> "1", + "spark.executor.instances" -> "1" + ) + val dataprocWorkerInfo = buildGpuWorkerInfoAsString(None, Some(32), + Some("212992MiB"), Some(5), Some(4), Some(T4Gpu.getMemory), Some(T4Gpu.toString)) + val infoProvider = getMockInfoProvider(0, Seq(0), Seq(0.0), + logEventsProps, Some(testSparkVersion)) + val clusterPropsOpt = ProfilingAutoTunerConfigsProvider + .loadClusterPropertiesFromContent(dataprocWorkerInfo) + val platform = PlatformFactory.createInstance(PlatformNames.DATAPROC, clusterPropsOpt) + val autoTuner: AutoTuner = ProfilingAutoTunerConfigsProvider + .buildAutoTunerFromProps(dataprocWorkerInfo, infoProvider, platform) + val (properties, comments) = autoTuner.getRecommendedProperties() + val autoTunerOutput = Profiler.getAutoTunerResultsAsString(properties, comments) + val expectedResults = Seq( + "--conf spark.dataproc.enhanced.optimizer.enabled=true", + "--conf spark.dataproc.enhanced.execution.enabled=true" + ) + assert(expectedResults.forall(autoTunerOutput.contains)) + } } From 08f9f5a77da32572a4c73b87d40a5fe310d94c8b Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 7 Feb 2025 17:35:03 -0800 Subject: [PATCH 2/4] Fix unit tests Signed-off-by: Partho Sarthi --- .../tool/tuning/BaseAutoTunerSuite.scala | 4 + .../tool/tuning/ProfilingAutoTunerSuite.scala | 190 +++++++++++++++++- 2 files changed, 184 insertions(+), 10 deletions(-) diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/BaseAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/BaseAutoTunerSuite.scala index acb51b826..39d431ae0 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/BaseAutoTunerSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/BaseAutoTunerSuite.scala @@ -71,8 +71,12 @@ abstract class BaseAutoTunerSuite extends FunSuite with BeforeAndAfterEach with // Spark runtime version used for testing def testSparkVersion: String = ToolUtils.sparkRuntimeVersion + // Databricks version used for testing + def testDatabricksVersion: String = "12.2.x-aarch64-scala2.12" // RapidsShuffleManager version used for testing def testSmVersion: String = testSparkVersion.filterNot(_ == '.') + // RapidsShuffleManager version used for testing Databricks + def testSmVersionDatabricks: String = "332db" val defaultDataprocProps: mutable.Map[String, String] = { mutable.LinkedHashMap[String, String]( diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala index 855654822..5e9fc7c83 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala @@ -76,6 +76,8 @@ class ProfilingAutoTunerSuite extends BaseAutoTunerSuite { val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=2 |--conf spark.executor.memory=32768m @@ -98,6 +100,8 @@ class ProfilingAutoTunerSuite extends BaseAutoTunerSuite { |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.cores' was not set. |- 'spark.executor.instances' was not set. |- 'spark.executor.memory' was not set. @@ -262,6 +266,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=8 |--conf spark.executor.memory=32768m @@ -284,6 +290,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.cores' was not set. |- 'spark.executor.instances' was not set. |- 'spark.executor.memory' was not set. @@ -324,6 +332,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=2 |--conf spark.executor.memory=32768m @@ -346,6 +356,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.cores' was not set. |- 'spark.executor.instances' was not set. |- 'spark.executor.memory' was not set. @@ -398,6 +410,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=4 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -414,6 +428,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. @@ -461,6 +477,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -476,6 +494,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -520,6 +540,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -533,6 +555,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE] | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -573,6 +597,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -588,6 +614,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -631,6 +659,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -646,6 +676,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -677,6 +709,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=2 |--conf spark.executor.memory=32768m @@ -699,6 +733,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.cores' was not set. |- 'spark.executor.instances' was not set. |- 'spark.executor.memory' was not set. @@ -744,6 +780,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -759,6 +797,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -801,6 +841,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=4 |--conf spark.executor.memoryOverhead=17612m @@ -817,6 +859,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -843,6 +887,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.cores=16 |--conf spark.executor.instances=8 |--conf spark.executor.memory=32768m @@ -865,6 +911,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.cores' was not set. |- 'spark.executor.instances' was not set. |- 'spark.executor.memory' was not set. @@ -945,6 +993,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -964,6 +1014,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1023,6 +1075,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=5 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1042,6 +1096,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1092,6 +1148,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1113,6 +1171,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. @@ -1172,6 +1232,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1193,6 +1255,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. @@ -1256,6 +1320,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1275,6 +1341,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1336,6 +1404,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1355,6 +1425,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1411,6 +1483,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1430,6 +1504,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1479,6 +1555,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -1494,6 +1572,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -1524,6 +1604,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -1539,6 +1621,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -1566,6 +1650,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=8 |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 @@ -1581,6 +1667,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -1633,6 +1721,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1653,6 +1743,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.filecache.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. @@ -1710,6 +1802,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -1729,6 +1823,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -1808,6 +1904,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.locality.wait=0 @@ -1826,6 +1924,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -2220,6 +2320,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2239,6 +2341,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=4096m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. @@ -2351,10 +2455,10 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val dataprocWorkerInfo = buildGpuWorkerInfoAsString(Some(customProps), Some(32), Some("212992MiB"), Some(5), Some(4), Some(T4Gpu.getMemory), Some(T4Gpu.toString)) val infoProvider = getMockInfoProvider(3.7449728E7, Seq(0, 0), Seq(0.4, 0.4), logEventsProps, - Some(testSparkVersion)) + Some(testDatabricksVersion)) val clusterPropsOpt = ProfilingAutoTunerConfigsProvider .loadClusterPropertiesFromContent(dataprocWorkerInfo) - val platform = PlatformFactory.createInstance(PlatformNames.DATAPROC, clusterPropsOpt) + val platform = PlatformFactory.createInstance(PlatformNames.DATABRICKS_AWS, clusterPropsOpt) val autoTuner: AutoTuner = ProfilingAutoTunerConfigsProvider .buildAutoTunerFromProps(dataprocWorkerInfo, infoProvider, platform) val (properties, comments) = autoTuner.getRecommendedProperties() @@ -2364,9 +2468,7 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. s"""| |Spark Properties: |--conf spark.databricks.adaptive.autoOptimizeShuffle.enabled=false - |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m - |--conf spark.executor.memoryOverhead=17612m |--conf spark.locality.wait=0 |--conf spark.rapids.memory.pinnedPool.size=4096m |--conf spark.rapids.shuffle.multiThreaded.maxBytesInFlight=4g @@ -2377,13 +2479,13 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.rapids.sql.format.parquet.multithreaded.combine.waitTime=1000 |--conf spark.rapids.sql.multiThreadedRead.numThreads=80 |--conf spark.rapids.sql.reader.multithreaded.combine.sizeBytes=10485760 + |--conf spark.shuffle.manager=com.nvidia.spark.rapids.spark$testSmVersionDatabricks.RapidsShuffleManager |--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=128m |--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE] |--conf spark.sql.files.maxPartitionBytes=3669m |--conf spark.task.resource.gpu.amount=0.001 | |Comments: - |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. @@ -2573,6 +2675,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2593,6 +2697,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.shuffle.partitions=400 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -2652,6 +2758,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2671,6 +2779,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.sql.files.maxPartitionBytes=3669m | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. |- 'spark.rapids.sql.batchSizeBytes' was not set. @@ -2718,6 +2828,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2741,6 +2853,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.kryo.registrator' was not set. |- 'spark.rapids.memory.pinnedPool.size' was not set. @@ -2794,6 +2908,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2817,6 +2933,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.memory.pinnedPool.size' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. @@ -2869,6 +2987,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. val expectedResults = s"""| |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true |--conf spark.executor.instances=10 |--conf spark.executor.memory=32768m |--conf spark.executor.memoryOverhead=17612m @@ -2892,6 +3012,8 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.task.resource.gpu.amount=0.001 | |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. |- 'spark.executor.memoryOverhead' was not set. |- 'spark.rapids.memory.pinnedPool.size' was not set. |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. @@ -3011,10 +3133,58 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. .buildAutoTunerFromProps(dataprocWorkerInfo, infoProvider, platform) val (properties, comments) = autoTuner.getRecommendedProperties() val autoTunerOutput = Profiler.getAutoTunerResultsAsString(properties, comments) - val expectedResults = Seq( - "--conf spark.dataproc.enhanced.optimizer.enabled=true", - "--conf spark.dataproc.enhanced.execution.enabled=true" - ) - assert(expectedResults.forall(autoTunerOutput.contains)) + // scalastyle:off line.size.limit + val expectedResults = + s"""| + |Spark Properties: + |--conf spark.dataproc.enhanced.execution.enabled=true + |--conf spark.dataproc.enhanced.optimizer.enabled=true + |--conf spark.executor.instances=10 + |--conf spark.executor.memory=32768m + |--conf spark.executor.memoryOverhead=17612m + |--conf spark.locality.wait=0 + |--conf spark.rapids.memory.pinnedPool.size=4096m + |--conf spark.rapids.shuffle.multiThreaded.maxBytesInFlight=4g + |--conf spark.rapids.shuffle.multiThreaded.reader.threads=28 + |--conf spark.rapids.shuffle.multiThreaded.writer.threads=28 + |--conf spark.rapids.sql.batchSizeBytes=2147483647 + |--conf spark.rapids.sql.concurrentGpuTasks=2 + |--conf spark.rapids.sql.enabled=true + |--conf spark.rapids.sql.format.parquet.multithreaded.combine.waitTime=1000 + |--conf spark.rapids.sql.multiThreadedRead.numThreads=80 + |--conf spark.rapids.sql.reader.multithreaded.combine.sizeBytes=10485760 + |--conf spark.shuffle.manager=com.nvidia.spark.rapids.spark350.RapidsShuffleManager + |--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=128m + |--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE] + |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m + |--conf spark.sql.files.maxPartitionBytes=512m + |--conf spark.task.resource.gpu.amount=0.001 + | + |Comments: + |- 'spark.dataproc.enhanced.execution.enabled' was not set. + |- 'spark.dataproc.enhanced.optimizer.enabled' was not set. + |- 'spark.executor.memoryOverhead' was not set. + |- 'spark.rapids.memory.pinnedPool.size' was not set. + |- 'spark.rapids.shuffle.multiThreaded.maxBytesInFlight' was not set. + |- 'spark.rapids.shuffle.multiThreaded.reader.threads' was not set. + |- 'spark.rapids.shuffle.multiThreaded.writer.threads' was not set. + |- 'spark.rapids.sql.batchSizeBytes' was not set. + |- 'spark.rapids.sql.concurrentGpuTasks' was not set. + |- 'spark.rapids.sql.enabled' was not set. + |- 'spark.rapids.sql.format.parquet.multithreaded.combine.waitTime' was not set. + |- 'spark.rapids.sql.multiThreadedRead.numThreads' was not set. + |- 'spark.rapids.sql.reader.multithreaded.combine.sizeBytes' was not set. + |- 'spark.shuffle.manager' was not set. + |- 'spark.sql.adaptive.advisoryPartitionSizeInBytes' was not set. + |- 'spark.sql.adaptive.autoBroadcastJoinThreshold' was not set. + |- 'spark.sql.adaptive.enabled' should be enabled for better performance. + |- 'spark.sql.files.maxPartitionBytes' was not set. + |- 'spark.task.resource.gpu.amount' was not set. + |- RAPIDS Accelerator for Apache Spark jar is missing in "spark.plugins". Please refer to https://docs.nvidia.com/spark-rapids/user-guide/latest/getting-started/overview.html + |- ${ProfilingAutoTunerConfigsProvider.classPathComments("rapids.jars.missing")} + |- ${ProfilingAutoTunerConfigsProvider.classPathComments("rapids.shuffle.jars")} + |""".stripMargin + // scalastyle:on line.size.limit + compareOutput(expectedResults, autoTunerOutput) } } From deca7430376d1400300c181030dbe1fa79984f74 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 7 Feb 2025 17:56:25 -0800 Subject: [PATCH 3/4] Add configs to tuning table yaml file Signed-off-by: Partho Sarthi --- core/src/main/resources/bootstrap/tuningTable.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/main/resources/bootstrap/tuningTable.yaml b/core/src/main/resources/bootstrap/tuningTable.yaml index a40c79611..199367cd4 100644 --- a/core/src/main/resources/bootstrap/tuningTable.yaml +++ b/core/src/main/resources/bootstrap/tuningTable.yaml @@ -18,6 +18,16 @@ tuningDefinitions: enabled: true level: job category: tuning + - label: spark.dataproc.enhanced.execution.enabled + description: 'Enables enhanced execution. It is recommended to turn it on for better performance on Dataproc.' + enabled: true + level: job + category: tuning + - label: spark.dataproc.enhanced.optimizer.enabled + description: 'Enables enhanced optimizer. It is recommended to turn it on for better performance on Dataproc.' + enabled: true + level: job + category: tuning - label: spark.executor.cores description: 'The number of cores to use on each executor. It is recommended to be set to 16' enabled: true From 3243602fb42052ef0961c8361d7c3c3e3d3ba614 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 7 Feb 2025 18:05:36 -0800 Subject: [PATCH 4/4] Add dynamic shuffle manager version Signed-off-by: Partho Sarthi --- .../spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala index 5e9fc7c83..36b5e9e8c 100644 --- a/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala +++ b/core/src/test/scala/com/nvidia/spark/rapids/tool/tuning/ProfilingAutoTunerSuite.scala @@ -3153,7 +3153,7 @@ We recommend using nodes/workers with more memory. Need at least 17496MB memory. |--conf spark.rapids.sql.format.parquet.multithreaded.combine.waitTime=1000 |--conf spark.rapids.sql.multiThreadedRead.numThreads=80 |--conf spark.rapids.sql.reader.multithreaded.combine.sizeBytes=10485760 - |--conf spark.shuffle.manager=com.nvidia.spark.rapids.spark350.RapidsShuffleManager + |--conf spark.shuffle.manager=com.nvidia.spark.rapids.spark$testSmVersion.RapidsShuffleManager |--conf spark.sql.adaptive.advisoryPartitionSizeInBytes=128m |--conf spark.sql.adaptive.autoBroadcastJoinThreshold=[FILL_IN_VALUE] |--conf spark.sql.adaptive.coalescePartitions.minPartitionSize=4m