Merge dev into main

Signed-off-by: spark-rapids automation <[email protected]>
NVIDIA · Dec 19, 2023 · 21e8e5e · 21e8e5e
2 parents cf9b5e0 + b54ae84
commit 21e8e5e
Show file tree

Hide file tree

Showing 55 changed files with 1,343 additions and 775 deletions.
diff --git a/.github/workflows/markdown-links-check.yml b/.github/workflows/markdown-links-check.yml
@@ -0,0 +1,36 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to check if PR got broken hyperlinks
+name: Check Markdown links
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  markdown-link-check:
+    runs-on: ubuntu-latest
+    steps:
+    - name: work around permission issue
+      run: git config --global --add safe.directory /github/workspace
+    - uses: actions/checkout@master
+    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+      with:
+        max-depth: -1
+        use-verbose-mode: 'yes'
+        config-file: '.github/workflows/markdown-links-check/markdown-links-check-config.json'
+        base-branch: 'dev'
+
+
diff --git a/.github/workflows/markdown-links-check/markdown-links-check-config.json b/.github/workflows/markdown-links-check/markdown-links-check-config.json
@@ -0,0 +1,17 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "https://github.com/NVIDIA/spark-rapids-tools/issues/*"
+    },
+    {
+      "pattern": "http://localhost*"
+    },
+    {
+      "pattern": "https://www.nvidia.com/en-us/security/pgp-key"
+    }
+  ],
+  "timeout": "15s",
+  "retryOn429": true,
+  "retryCount":30,
+  "aliveStatusCodes": [200, 403]
+}
diff --git a/README.md b/README.md
@@ -4,12 +4,15 @@ This repo provides the tools to use [RAPIDS Accelerator for Apache Spark](https:
 
 ## Catalog
 
-- [RAPIDS core tools](/core): Tools that help developers getting the most out of their Apache Spark applications
+- [RAPIDS core tools](./core): Tools that help developers getting the most out of their Apache 
+  Spark applications
   without any code change:
   - Report acceleration potential of RAPIDS Accelerator for Apache Spark on a set of Spark applications.
   - Generate comprehensive profiling analysis for Apache Sparks executing on accelerated GPU instances. This information
     can be used to further tune and optimize the application.
-- [spark-rapids-user-tools](/user_tools): A simple wrapper process around cloud service providers to run
-  [RAPIDS core tools](/core) across multiple cloud platforms. In addition, the output educates the users on
+- [spark-rapids-user-tools](./user_tools): A simple wrapper process around cloud service 
+  providers to run
+  [RAPIDS core tools](./core) across multiple cloud platforms. In addition, the output educates 
+  the users on
   the cost savings and acceleration potential of RAPIDS Accelerator for Apache Spark and makes recommendations to tune
   the application performance based on the cluster shape.
diff --git a/core/pom.xml b/core/pom.xml
@@ -23,7 +23,7 @@
     <artifactId>rapids-4-spark-tools_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark tools</name>
     <description>RAPIDS Accelerator for Apache Spark tools</description>
-    <version>23.10.1</version>
+    <version>23.10.2-SNAPSHOT</version>
     <packaging>jar</packaging>
     <url>http://github.com/NVIDIA/spark-rapids-tools</url>
 

diff --git a/core/src/main/resources/operatorsScore-databricks-aws.csv b/core/src/main/resources/operatorsScore-databricks-aws.csv
@@ -105,6 +105,7 @@ Expm1,2.45
 First,2.45
 Flatten,2.45
 Floor,2.45
+FormatNumber,2.45
 FromUTCTimestamp,2.45
 FromUnixTime,2.45
 GetArrayItem,2.45
@@ -175,6 +176,7 @@ Not,2.45
 NthValue,2.45
 OctetLength,2.45
 Or,2.45
+Percentile,2.45
 PercentRank,2.45
 PivotFirst,2.45
 Pmod,2.45
@@ -214,6 +216,7 @@ SortOrder,2.45
 SparkPartitionID,2.45
 SpecifiedWindowFrame,2.45
 Sqrt,2.45
+Stack,2.45
 StartsWith,2.45
 StddevPop,2.45
 StddevSamp,2.45
@@ -229,6 +232,7 @@ StringTranslate,2.45
 StringTrim,2.45
 StringTrimLeft,2.45
 StringTrimRight,2.45
+StructsToJson,2.45
 Substring,2.45
 SubstringIndex,2.45
 Subtract,2.45

diff --git a/core/src/main/resources/operatorsScore-databricks-azure.csv b/core/src/main/resources/operatorsScore-databricks-azure.csv
@@ -105,6 +105,7 @@ Expm1,2.73
 First,2.73
 Flatten,2.73
 Floor,2.73
+FormatNumber,2.73
 FromUTCTimestamp,2.73
 FromUnixTime,2.73
 GetArrayItem,2.73
@@ -175,6 +176,7 @@ Not,2.73
 NthValue,2.73
 OctetLength,2.73
 Or,2.73
+Percentile,2.73
 PercentRank,2.73
 PivotFirst,2.73
 Pmod,2.73
@@ -214,6 +216,7 @@ SortOrder,2.73
 SparkPartitionID,2.73
 SpecifiedWindowFrame,2.73
 Sqrt,2.73
+Stack,2.73
 StartsWith,2.73
 StddevPop,2.73
 StddevSamp,2.73
@@ -229,6 +232,7 @@ StringTranslate,2.73
 StringTrim,2.73
 StringTrimLeft,2.73
 StringTrimRight,2.73
+StructsToJson,2.73
 Substring,2.73
 SubstringIndex,2.73
 Subtract,2.73

diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv
@@ -98,6 +98,7 @@ Expm1,3.74
 First,3.74
 Flatten,3.74
 Floor,3.74
+FormatNumber,3.74
 FromUTCTimestamp,3.74
 FromUnixTime,3.74
 GetArrayItem,3.74
@@ -168,6 +169,7 @@ Not,3.74
 NthValue,3.74
 OctetLength,3.74
 Or,3.74
+Percentile,3.74
 PercentRank,3.74
 PivotFirst,3.74
 Pmod,3.74
@@ -207,6 +209,7 @@ SortOrder,3.74
 SparkPartitionID,3.74
 SpecifiedWindowFrame,3.74
 Sqrt,3.74
+Stack,3.74
 StartsWith,3.74
 StddevPop,3.74
 StddevSamp,3.74
@@ -222,6 +225,7 @@ StringTranslate,3.74
 StringTrim,3.74
 StringTrimLeft,3.74
 StringTrimRight,3.74
+StructsToJson,3.74
 Substring,3.74
 SubstringIndex,3.74
 Subtract,3.74

diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv
@@ -98,6 +98,7 @@ Expm1,3.65
 First,3.65
 Flatten,3.65
 Floor,3.65
+FormatNumber,3.65
 FromUTCTimestamp,3.65
 FromUnixTime,3.65
 GetArrayItem,3.65
@@ -168,6 +169,7 @@ Not,3.65
 NthValue,3.65
 OctetLength,3.65
 Or,3.65
+Percentile,3.65
 PercentRank,3.65
 PivotFirst,3.65
 Pmod,3.65
@@ -207,6 +209,7 @@ SortOrder,3.65
 SparkPartitionID,3.65
 SpecifiedWindowFrame,3.65
 Sqrt,3.65
+Stack,3.65
 StartsWith,3.65
 StddevPop,3.65
 StddevSamp,3.65
@@ -222,6 +225,7 @@ StringTranslate,3.65
 StringTrim,3.65
 StringTrimLeft,3.65
 StringTrimRight,3.65
+StructsToJson,3.65
 Substring,3.65
 SubstringIndex,3.65
 Subtract,3.65

diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv
@@ -105,6 +105,7 @@ Expm1,4.16
 First,4.16
 Flatten,4.16
 Floor,4.16
+FormatNumber,4.16
 FromUTCTimestamp,4.16
 FromUnixTime,4.16
 GetArrayItem,4.16
@@ -175,6 +176,7 @@ Not,4.16
 NthValue,4.16
 OctetLength,4.16
 Or,4.16
+Percentile,4.16
 PercentRank,4.16
 PivotFirst,4.16
 Pmod,4.16
@@ -214,6 +216,7 @@ SortOrder,4.16
 SparkPartitionID,4.16
 SpecifiedWindowFrame,4.16
 Sqrt,4.16
+Stack,4.16
 StartsWith,4.16
 StddevPop,4.16
 StddevSamp,4.16
@@ -229,6 +232,7 @@ StringTranslate,4.16
 StringTrim,4.16
 StringTrimLeft,4.16
 StringTrimRight,4.16
+StructsToJson,4.16
 Substring,4.16
 SubstringIndex,4.16
 Subtract,4.16

diff --git a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv
@@ -98,6 +98,7 @@ Expm1,4.25
 First,4.25
 Flatten,4.25
 Floor,4.25
+FormatNumber,4.25
 FromUTCTimestamp,4.25
 FromUnixTime,4.25
 GetArrayItem,4.25
@@ -168,6 +169,7 @@ Not,4.25
 NthValue,4.25
 OctetLength,4.25
 Or,4.25
+Percentile,4.25
 PercentRank,4.25
 PivotFirst,4.25
 Pmod,4.25
@@ -207,6 +209,7 @@ SortOrder,4.25
 SparkPartitionID,4.25
 SpecifiedWindowFrame,4.25
 Sqrt,4.25
+Stack,4.25
 StartsWith,4.25
 StddevPop,4.25
 StddevSamp,4.25
@@ -222,6 +225,7 @@ StringTranslate,4.25
 StringTrim,4.25
 StringTrimLeft,4.25
 StringTrimRight,4.25
+StructsToJson,4.25
 Substring,4.25
 SubstringIndex,4.25
 Subtract,4.25

diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv
@@ -105,6 +105,7 @@ Expm1,4.88
 First,4.88
 Flatten,4.88
 Floor,4.88
+FormatNumber,4.88
 FromUTCTimestamp,4.88
 FromUnixTime,4.88
 GetArrayItem,4.88
@@ -175,6 +176,7 @@ Not,4.88
 NthValue,4.88
 OctetLength,4.88
 Or,4.88
+Percentile,4.88
 PercentRank,4.88
 PivotFirst,4.88
 Pmod,4.88
@@ -214,6 +216,7 @@ SortOrder,4.88
 SparkPartitionID,4.88
 SpecifiedWindowFrame,4.88
 Sqrt,4.88
+Stack,4.88
 StartsWith,4.88
 StddevPop,4.88
 StddevSamp,4.88
@@ -229,6 +232,7 @@ StringTranslate,4.88
 StringTrim,4.88
 StringTrimLeft,4.88
 StringTrimRight,4.88
+StructsToJson,4.88
 Substring,4.88
 SubstringIndex,4.88
 Subtract,4.88

diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv
@@ -105,6 +105,7 @@ Expm1,2.59
 First,2.59
 Flatten,2.59
 Floor,2.59
+FormatNumber,2.59
 FromUTCTimestamp,2.59
 FromUnixTime,2.59
 GetArrayItem,2.59
@@ -175,6 +176,7 @@ Not,2.59
 NthValue,2.59
 OctetLength,2.59
 Or,2.59
+Percentile,2.59
 PercentRank,2.59
 PivotFirst,2.59
 Pmod,2.59
@@ -214,6 +216,7 @@ SortOrder,2.59
 SparkPartitionID,2.59
 SpecifiedWindowFrame,2.59
 Sqrt,2.59
+Stack,2.59
 StartsWith,2.59
 StddevPop,2.59
 StddevSamp,2.59
@@ -229,6 +232,7 @@ StringTranslate,2.59
 StringTrim,2.59
 StringTrimLeft,2.59
 StringTrimRight,2.59
+StructsToJson,2.59
 Substring,2.59
 SubstringIndex,2.59
 Subtract,2.59

diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv
@@ -105,6 +105,7 @@ Expm1,2.07
 First,2.07
 Flatten,2.07
 Floor,2.07
+FormatNumber,2.07
 FromUTCTimestamp,2.07
 FromUnixTime,2.07
 GetArrayItem,2.07
@@ -175,6 +176,7 @@ Not,2.07
 NthValue,2.07
 OctetLength,2.07
 Or,2.07
+Percentile,2.07
 PercentRank,2.07
 PivotFirst,2.07
 Pmod,2.07
@@ -214,6 +216,7 @@ SortOrder,2.07
 SparkPartitionID,2.07
 SpecifiedWindowFrame,2.07
 Sqrt,2.07
+Stack,2.07
 StartsWith,2.07
 StddevPop,2.07
 StddevSamp,2.07
@@ -229,6 +232,7 @@ StringTranslate,2.07
 StringTrim,2.07
 StringTrimLeft,2.07
 StringTrimRight,2.07
+StructsToJson,2.07
 Substring,2.07
 SubstringIndex,2.07
 Subtract,2.07