Merge dev into main

Signed-off-by: spark-rapids automation <[email protected]>
NVIDIA · Mar 27, 2024 · 2c1561e · 2c1561e
2 parents 756ed54 + b1916ea
commit 2c1561e
Show file tree

Hide file tree

Showing 75 changed files with 4,653 additions and 1,419 deletions.
diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,7 +28,7 @@ jobs:
     name: Add new issues and pull requests to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v0.6.1
         with:
           project-url: https://github.com/orgs/NVIDIA/projects/4
           github-token: ${{ secrets.PROJECT_TOKEN }}

diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,12 +25,12 @@ jobs:
     strategy:
       matrix:
         java-version: [8, 11]
-        spark-version: ['311', '320', '333', '341']
+        spark-version: ['314', '325', '334', '350', '400']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Setup Java
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: adopt
         java-version: ${{ matrix.java-version }}

diff --git a/.github/workflows/python-unit-test.yml b/.github/workflows/python-unit-test.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,10 +27,10 @@ jobs:
         python-version: ['3.8', '3.9', '3.10']
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,3 +1,17 @@
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: Release
 
 on:
@@ -16,7 +30,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.NVAUTO_TOKEN }}
 

diff --git a/.github/workflows/signoff-check.yml b/.github/workflows/signoff-check.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: sigoff-check job
         uses: ./.github/workflows/signoff-check

diff --git a/.gitignore b/.gitignore
@@ -51,3 +51,6 @@ csp-resources*
 
 # precommit files
 *-E
+
+# ignore report generated by the plugin sync scripts
+**/operators_plugin_sync_report.txt
diff --git a/core/README.md b/core/README.md
@@ -20,14 +20,14 @@ mvn clean package
 ```
 
 After a successful build, the jar of 'rapids-4-spark-tools_2.12-*-SNAPSHOT.jar' will be in 'target/' directory.  
-This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.3.3.
+This will build the plugin for a single version of Spark. By default, this is Apache Spark 3.5.0.
 
 For development purpose, you may need to run the tests against different spark versions.
 To run the tests against a specific Spark version, you can use the `-Dbuildver=XXX` command line option.  
-For instance to build Spark 3.4.1 you would use:
+For instance to build Spark 3.5.1 you would use:
 
 ```shell script
-mvn -Dbuildver=341 clean package
+mvn -Dbuildver=351 clean package
 ```
 
 Run `mvn help:all-profiles` to list supported Spark versions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -23,7 +23,7 @@
     <artifactId>rapids-4-spark-tools_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark tools</name>
     <description>RAPIDS Accelerator for Apache Spark tools</description>
-    <version>24.02.1</version>
+    <version>24.02.2-SNAPSHOT</version>
     <packaging>jar</packaging>
     <url>http://github.com/NVIDIA/spark-rapids-tools</url>
 
@@ -268,7 +268,6 @@
         <profile>
             <id>release333</id>
             <activation>
-                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>333</value>
@@ -296,6 +295,21 @@
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
+        <profile>
+            <id>release335</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>335</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>335</buildver>
+                <spark.version>${spark335.version}</spark.version>
+                <delta.core.version>${delta23x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
         <profile>
             <id>release340</id>
             <activation>
@@ -341,9 +355,25 @@
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
+        <profile>
+            <id>release343</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>343</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>343</buildver>
+                <spark.version>${spark343.version}</spark.version>
+                <delta.core.version>${delta24x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
         <profile>
             <id>release350</id>
             <activation>
+                <activeByDefault>true</activeByDefault>
                 <property>
                     <name>buildver</name>
                     <value>350</value>
@@ -352,7 +382,7 @@
             <properties>
                 <buildver>350</buildver>
                 <spark.version>${spark350.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -367,7 +397,22 @@
             <properties>
                 <buildver>351</buildver>
                 <spark.version>${spark351.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
+                <hadoop.version>3.3.6</hadoop.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>release352</id>
+            <activation>
+                <property>
+                    <name>buildver</name>
+                    <value>352</value>
+                </property>
+            </activation>
+            <properties>
+                <buildver>352</buildver>
+                <spark.version>${spark352.version}</spark.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -382,7 +427,7 @@
             <properties>
                 <buildver>400</buildver>
                 <spark.version>${spark400.version}</spark.version>
-                <delta.core.version>${delta24x.version}</delta.core.version>
+                <delta.core.version>${delta31x.version}</delta.core.version>
                 <hadoop.version>3.3.6</hadoop.version>
             </properties>
         </profile>
@@ -402,12 +447,15 @@
         <spark331.version>3.3.1</spark331.version>
         <spark332.version>3.3.2</spark332.version>
         <spark333.version>3.3.3</spark333.version>
-        <spark334.version>3.3.4-SNAPSHOT</spark334.version>
+        <spark334.version>3.3.4</spark334.version>
+        <spark335.version>3.3.5-SNAPSHOT</spark335.version>
         <spark340.version>3.4.0</spark340.version>
         <spark341.version>3.4.1</spark341.version>
-        <spark342.version>3.4.2-SNAPSHOT</spark342.version>
-        <spark350.version>3.5.0-SNAPSHOT</spark350.version>
-        <spark351.version>3.5.1-SNAPSHOT</spark351.version>
+        <spark342.version>3.4.2</spark342.version>
+        <spark343.version>3.4.3-SNAPSHOT</spark343.version>
+        <spark350.version>3.5.0</spark350.version>
+        <spark351.version>3.5.1</spark351.version>
+        <spark352.version>3.5.2-SNAPSHOT</spark352.version>
         <spark400.version>4.0.0-SNAPSHOT</spark400.version>
         <scala.binary.version>2.12</scala.binary.version>
         <scala.plugin.version>4.3.0</scala.plugin.version>
@@ -439,7 +487,10 @@
         <delta22x.version>2.2.0</delta22x.version>
         <delta23x.version>2.3.0</delta23x.version>
         <delta24x.version>2.4.0</delta24x.version>
-        <delta.core.version>${delta10x.version}</delta.core.version>
+        <!-- TODO: Fix delta version should be 3.1.0 but this requires scala-binary 2.13 which we
+                   don't have yet -->
+        <delta31x.version>2.4.0</delta31x.version>
+        <delta.core.version>${delta24x.version}</delta.core.version>
         <!-- environment properties -->
         <java.version>1.8</java.version>
         <platform-encoding>UTF-8</platform-encoding>

diff --git a/core/src/main/resources/supportedDataSource.csv b/core/src/main/resources/supportedDataSource.csv
@@ -1,13 +1,13 @@
-Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT
-Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO
-CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA
-Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS
-Delta,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
-HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS
-HiveText,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
-Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS
-JSON,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO
-ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS
-ORC,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
-Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS
-Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
+Format,Direction,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH
+Avro,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO
+CSV,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,NA,NA,NA,NA,NS,NS
+Delta,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S
+Delta,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS
+HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS
+HiveText,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS
+Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S
+JSON,read,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO
+ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NS,NS
+ORC,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS
+Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S
+Parquet,write,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,NS