Update org.apache.hadoop:* to v3.4.0 (#2380)

* Update org.apache.hadoop:* to v3.4.0 * Exclude jline and jsch * Fix lock for ReplicaMap * Use IOUtils.cleanupWithLogger insteaad of deprecated IOUtils.cleanup * unusedDeclaredArtifacts org.slf4j:jcl-over-slf4j:2.0.13@jar Co-authored-by: Jan Høydahl <[email protected]>
apache · Oct 17, 2024 · fef5f5d · fef5f5d
1 parent 0f4a465
commit fef5f5d
Show file tree

Hide file tree

Showing 50 changed files with 52 additions and 51 deletions.
diff --git a/solr/licenses/hadoop-annotations-3.3.6.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-annotations-3.4.0.jar.sha1 b/solr/licenses/hadoop-annotations-3.4.0.jar.sha1
@@ -0,0 +1 @@
+612302e6e385d7e6a62bc0129dbc58689f8c7b2b
diff --git a/solr/licenses/hadoop-auth-3.3.6.jar.sha1 b/solr/licenses/hadoop-auth-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-auth-3.4.0.jar.sha1 b/solr/licenses/hadoop-auth-3.4.0.jar.sha1
@@ -0,0 +1 @@
+6f5ab3a46b3d2734c0ae4d6338f0be3529efe083
diff --git a/solr/licenses/hadoop-client-api-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-client-api-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-api-3.4.0.jar.sha1
@@ -0,0 +1 @@
+14b26fbacedff53bc10eebc1fddface4f1f655af
diff --git a/solr/licenses/hadoop-client-minicluster-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-client-minicluster-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.4.0.jar.sha1
@@ -0,0 +1 @@
+a1af909b1cb220c0f81224b2241c4a0aa2860ee6
diff --git a/solr/licenses/hadoop-client-runtime-3.3.6.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-client-runtime-3.4.0.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.4.0.jar.sha1
@@ -0,0 +1 @@
+de1ed42503ec64b84a91b330fdd2ed508b6f9df8
diff --git a/solr/licenses/hadoop-common-3.3.6.jar.sha1 b/solr/licenses/hadoop-common-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-common-3.4.0.jar.sha1 b/solr/licenses/hadoop-common-3.4.0.jar.sha1
@@ -0,0 +1 @@
+2b4b1694b695b31cdd8e345b6b59fd74d43a26bc
diff --git a/solr/licenses/hadoop-hdfs-3.3.6-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.6-tests.jar.sha1
diff --git a/solr/licenses/hadoop-hdfs-3.3.6.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-hdfs-3.4.0-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.4.0-tests.jar.sha1
@@ -0,0 +1 @@
+18f9797a908899efbe4e27f6d5b76420f446a695
diff --git a/solr/licenses/hadoop-hdfs-3.4.0.jar.sha1 b/solr/licenses/hadoop-hdfs-3.4.0.jar.sha1
@@ -0,0 +1 @@
+d46983ccf965bffaeab8b67713aa9442c5b7d5bf
diff --git a/solr/licenses/hadoop-minikdc-3.3.6.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.6.jar.sha1
diff --git a/solr/licenses/hadoop-minikdc-3.4.0.jar.sha1 b/solr/licenses/hadoop-minikdc-3.4.0.jar.sha1
@@ -0,0 +1 @@
+ec7c619e1deff5a9f2da0293fc9f5fcae66d159e
diff --git a/solr/licenses/kerb-admin-1.0.1.jar.sha1 b/solr/licenses/kerb-admin-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-admin-2.0.3.jar.sha1 b/solr/licenses/kerb-admin-2.0.3.jar.sha1
@@ -0,0 +1 @@
+e239e34968f59b4c7f75cffff08c6a140b3afa44
diff --git a/solr/licenses/kerb-client-1.0.1.jar.sha1 b/solr/licenses/kerb-client-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-client-2.0.3.jar.sha1 b/solr/licenses/kerb-client-2.0.3.jar.sha1
@@ -0,0 +1 @@
+ae6bb140ed545771efc13973f5ad665766a2e1d4
diff --git a/solr/licenses/kerb-common-1.0.1.jar.sha1 b/solr/licenses/kerb-common-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-common-2.0.3.jar.sha1 b/solr/licenses/kerb-common-2.0.3.jar.sha1
@@ -0,0 +1 @@
+32fde271604661ef07ec61b6654d381dfeaff5b1
diff --git a/solr/licenses/kerb-core-1.0.1.jar.sha1 b/solr/licenses/kerb-core-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-core-2.0.3.jar.sha1 b/solr/licenses/kerb-core-2.0.3.jar.sha1
@@ -0,0 +1 @@
+2ed4ee03585bb1749f3d5bf4951208cb36839e82
diff --git a/solr/licenses/kerb-crypto-1.0.1.jar.sha1 b/solr/licenses/kerb-crypto-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-crypto-2.0.3.jar.sha1 b/solr/licenses/kerb-crypto-2.0.3.jar.sha1
@@ -0,0 +1 @@
+a19d9c3fbcf52abd2f804f2d88f8bbf4195883a8
diff --git a/solr/licenses/kerb-identity-1.0.1.jar.sha1 b/solr/licenses/kerb-identity-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-identity-2.0.3.jar.sha1 b/solr/licenses/kerb-identity-2.0.3.jar.sha1
@@ -0,0 +1 @@
+0704d04804ff500f160b99ecf83258187e31d5ef
diff --git a/solr/licenses/kerb-server-1.0.1.jar.sha1 b/solr/licenses/kerb-server-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-server-2.0.3.jar.sha1 b/solr/licenses/kerb-server-2.0.3.jar.sha1
@@ -0,0 +1 @@
+7e2ea712dd36f8039a437d61379a60a388e1b2d2
diff --git a/solr/licenses/kerb-simplekdc-1.0.1.jar.sha1 b/solr/licenses/kerb-simplekdc-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-simplekdc-2.0.3.jar.sha1 b/solr/licenses/kerb-simplekdc-2.0.3.jar.sha1
@@ -0,0 +1 @@
+d267abb85f5aa3223e66b978c5b5a148d1fa5304
diff --git a/solr/licenses/kerb-util-1.0.1.jar.sha1 b/solr/licenses/kerb-util-1.0.1.jar.sha1
diff --git a/solr/licenses/kerb-util-2.0.3.jar.sha1 b/solr/licenses/kerb-util-2.0.3.jar.sha1
@@ -0,0 +1 @@
+dda5fbcbd1e8d5de40edcf6161faf94b915152ce
diff --git a/solr/licenses/kerby-asn1-1.0.1.jar.sha1 b/solr/licenses/kerby-asn1-1.0.1.jar.sha1
diff --git a/solr/licenses/kerby-asn1-2.0.3.jar.sha1 b/solr/licenses/kerby-asn1-2.0.3.jar.sha1
@@ -0,0 +1 @@
+71e4005a005c8c4476b959befdc1ad668fbbe758
diff --git a/solr/licenses/kerby-config-1.0.1.jar.sha1 b/solr/licenses/kerby-config-1.0.1.jar.sha1
diff --git a/solr/licenses/kerby-config-2.0.3.jar.sha1 b/solr/licenses/kerby-config-2.0.3.jar.sha1
@@ -0,0 +1 @@
+322b25b1fb7f57f6987c54c5ef32887036108d61
diff --git a/solr/licenses/kerby-pkix-1.0.1.jar.sha1 b/solr/licenses/kerby-pkix-1.0.1.jar.sha1
diff --git a/solr/licenses/kerby-pkix-2.0.3.jar.sha1 b/solr/licenses/kerby-pkix-2.0.3.jar.sha1
@@ -0,0 +1 @@
+8301fe828f3b931617df990f77da636eacd9212d
diff --git a/solr/licenses/kerby-util-1.0.1.jar.sha1 b/solr/licenses/kerby-util-1.0.1.jar.sha1
diff --git a/solr/licenses/kerby-util-2.0.3.jar.sha1 b/solr/licenses/kerby-util-2.0.3.jar.sha1
@@ -0,0 +1 @@
+f1e80545460c37fca6b2c18f02fa32f6040928f0
diff --git a/solr/modules/hadoop-auth/build.gradle b/solr/modules/hadoop-auth/build.gradle
@@ -99,6 +99,8 @@ dependencies {
     exclude group:'org.apache.kerby', module:'kerby-xdr'
     exclude group:'org.apache.kerby', module:'token-provider'
     exclude group:'org.slf4j', module:'slf4j-reload4j'
+    exclude group:'org.jline', module:'jline'
+    exclude group:'com.jcraft', module:'jsch'
   })
 
   // Zookeeper dependency - some tests like HdfsCloudBackupRestore need this

diff --git a/solr/modules/hdfs/build.gradle b/solr/modules/hdfs/build.gradle
@@ -50,7 +50,6 @@ dependencies {
   testImplementation 'org.apache.hadoop.thirdparty:hadoop-shaded-guava'
   testRuntimeOnly 'org.apache.hadoop:hadoop-client-minicluster'
 
-  testImplementation 'org.slf4j:jcl-over-slf4j'
   testRuntimeOnly 'org.apache.logging.log4j:log4j-1.2-api'
 
   // classes like solr.ICUCollationField, used by NNFailoverTest for example.

diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/RawLocalFileSystem.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -329,7 +329,7 @@ private LocalFSFileOutputStream(Path f, boolean append,
             success = true;
           } finally {
             if (!success) {
-              IOUtils.cleanup(LOG, this.fos);
+              IOUtils.cleanupWithLogger(LOG, this.fos);
             }
           }
         }

diff --git a/...s/hdfs/src/test/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/...s/hdfs/src/test/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
+import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.DataStorage;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
@@ -72,7 +73,6 @@
 import java.util.concurrent.RecursiveAction;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * A block pool slice represents a portion of a block pool stored on a volume.
@@ -787,7 +787,7 @@ void shutdown(BlockListAsLongs blocksListToPersist) {
 
   private boolean readReplicasFromCache(ReplicaMap volumeMap,
       final RamDiskReplicaTracker lazyWriteReplicaMap) {
-    ReplicaMap tmpReplicaMap = new ReplicaMap(new ReentrantReadWriteLock());
+    ReplicaMap tmpReplicaMap = new ReplicaMap(new DataSetLockManager());
     File replicaFile = new File(replicaCacheDir, REPLICA_CACHE_FILE);
     // Check whether the file exists or not.
     if (!replicaFile.exists()) {

diff --git a/versions.lock b/versions.lock
@@ -90,7 +90,7 @@ commons-cli:commons-cli:1.9.0 (2 constraints: a90fcab3)
 commons-codec:commons-codec:1.17.1 (11 constraints: 5a953ac4)
 commons-collections:commons-collections:3.2.2 (3 constraints: 77242dac)
 commons-digester:commons-digester:2.1 (1 constraints: 550fb664)
-commons-io:commons-io:2.15.1 (10 constraints: 47759e4f)
+commons-io:commons-io:2.15.1 (10 constraints: ff7571ba)
 commons-validator:commons-validator:1.7 (1 constraints: a10a3dd2)
 de.l3s.boilerpipe:boilerpipe:1.1.0 (1 constraints: 590ce401)
 edu.ucar:cdm:4.5.5 (3 constraints: 9d1abd7d)
@@ -197,11 +197,11 @@ org.apache.commons:commons-text:1.12.0 (2 constraints: 651f97e5)
 org.apache.curator:curator-client:5.7.0 (2 constraints: ec14cea3)
 org.apache.curator:curator-framework:5.7.0 (2 constraints: 0914ad75)
 org.apache.curator:curator-recipes:5.7.0 (1 constraints: 0e051936)
-org.apache.hadoop:hadoop-annotations:3.3.6 (1 constraints: 0e050936)
-org.apache.hadoop:hadoop-auth:3.3.6 (1 constraints: 0e050936)
-org.apache.hadoop:hadoop-client-api:3.3.6 (3 constraints: 25280861)
-org.apache.hadoop:hadoop-client-runtime:3.3.6 (2 constraints: 6f17dc43)
-org.apache.hadoop:hadoop-common:3.3.6 (1 constraints: 0e050936)
+org.apache.hadoop:hadoop-annotations:3.4.0 (1 constraints: 09050636)
+org.apache.hadoop:hadoop-auth:3.4.0 (1 constraints: 09050636)
+org.apache.hadoop:hadoop-client-api:3.4.0 (3 constraints: 16281b5e)
+org.apache.hadoop:hadoop-client-runtime:3.4.0 (2 constraints: 6517d242)
+org.apache.hadoop:hadoop-common:3.4.0 (1 constraints: 09050636)
 org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.2.0 (1 constraints: 0505f635)
 org.apache.httpcomponents:httpclient:4.5.14 (9 constraints: 62806342)
 org.apache.httpcomponents:httpcore:4.4.16 (8 constraints: 256d4617)
@@ -222,13 +222,13 @@ org.apache.kafka:kafka-storage-api:3.7.1 (2 constraints: 0218d3d8)
 org.apache.kafka:kafka-streams:3.7.1 (1 constraints: 0d051036)
 org.apache.kafka:kafka-tools-api:3.7.1 (1 constraints: 020b16e8)
 org.apache.kafka:kafka_2.13:3.7.1 (1 constraints: 0d051036)
-org.apache.kerby:kerb-core:1.0.1 (5 constraints: 683677dd)
-org.apache.kerby:kerb-crypto:1.0.1 (2 constraints: d617fe96)
-org.apache.kerby:kerb-util:1.0.1 (3 constraints: 1a1ddb4c)
-org.apache.kerby:kerby-asn1:1.0.1 (1 constraints: fd0be9f4)
-org.apache.kerby:kerby-config:1.0.1 (4 constraints: 4d3182b9)
-org.apache.kerby:kerby-pkix:1.0.1 (1 constraints: 710bfce4)
-org.apache.kerby:kerby-util:1.0.1 (2 constraints: 6518bdb6)
+org.apache.kerby:kerb-core:2.0.3 (5 constraints: 743627e0)
+org.apache.kerby:kerb-crypto:2.0.3 (2 constraints: dc177597)
+org.apache.kerby:kerb-util:2.0.3 (3 constraints: 201d584d)
+org.apache.kerby:kerby-asn1:2.0.3 (1 constraints: 000cf0f4)
+org.apache.kerby:kerby-config:2.0.3 (4 constraints: 593132bc)
+org.apache.kerby:kerby-pkix:2.0.3 (1 constraints: 740b03e5)
+org.apache.kerby:kerby-util:2.0.3 (2 constraints: 6b1837b7)
 org.apache.logging.log4j:log4j-1.2-api:2.21.0 (1 constraints: 3705363b)
 org.apache.logging.log4j:log4j-api:2.21.0 (10 constraints: f07d3453)
 org.apache.logging.log4j:log4j-core:2.21.0 (5 constraints: 5f510091)
@@ -380,14 +380,14 @@ org.scala-lang.modules:scala-java8-compat_2.13:1.0.2 (1 constraints: fa0af8e7)
 org.semver4j:semver4j:5.3.0 (1 constraints: 0a050d36)
 org.slf4j:jcl-over-slf4j:2.0.13 (3 constraints: fa17e8b5)
 org.slf4j:jul-to-slf4j:2.0.13 (3 constraints: 54285f5f)
-org.slf4j:slf4j-api:2.0.13 (71 constraints: 9dbf4f86)
+org.slf4j:slf4j-api:2.0.13 (71 constraints: a1bfff9b)
 org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528)
 org.tallison:jmatio:1.5 (1 constraints: ff0b57e9)
 org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28)
 org.tallison.xmp:xmpcore-shaded:6.1.10 (1 constraints: 300e8d49)
 org.threeten:threetenbp:1.6.9 (4 constraints: 2833ea68)
 org.tukaani:xz:1.9 (1 constraints: 030c5be9)
-org.xerial.snappy:snappy-java:1.1.10.5 (5 constraints: 3d469c91)
+org.xerial.snappy:snappy-java:1.1.10.5 (5 constraints: be46ae36)
 software.amazon.awssdk:annotations:2.26.19 (28 constraints: 48a79471)
 software.amazon.awssdk:apache-client:2.26.19 (4 constraints: 112adae3)
 software.amazon.awssdk:arns:2.26.19 (2 constraints: 231878c1)
@@ -451,15 +451,15 @@ net.bytebuddy:byte-buddy-agent:1.14.15 (1 constraints: 760bb5e9)
 net.minidev:accessors-smart:2.4.9 (1 constraints: 500a92b8)
 net.minidev:json-smart:2.4.10 (1 constraints: 400e9a7c)
 no.nav.security:mock-oauth2-server:0.5.10 (1 constraints: 3805333b)
-org.apache.hadoop:hadoop-client-minicluster:3.3.6 (1 constraints: 0e050936)
-org.apache.hadoop:hadoop-hdfs:3.3.6 (1 constraints: 0e050936)
-org.apache.hadoop:hadoop-minikdc:3.3.6 (1 constraints: 0e050936)
-org.apache.kerby:kerb-admin:1.0.1 (1 constraints: 840d892f)
-org.apache.kerby:kerb-client:1.0.1 (1 constraints: 840d892f)
-org.apache.kerby:kerb-common:1.0.1 (2 constraints: a51841ca)
-org.apache.kerby:kerb-identity:1.0.1 (1 constraints: 5f0cb602)
-org.apache.kerby:kerb-server:1.0.1 (1 constraints: d10b65f2)
-org.apache.kerby:kerb-simplekdc:1.0.1 (1 constraints: dc0d7e3e)
+org.apache.hadoop:hadoop-client-minicluster:3.4.0 (1 constraints: 09050636)
+org.apache.hadoop:hadoop-hdfs:3.4.0 (1 constraints: 09050636)
+org.apache.hadoop:hadoop-minikdc:3.4.0 (1 constraints: 09050636)
+org.apache.kerby:kerb-admin:2.0.3 (1 constraints: 870d902f)
+org.apache.kerby:kerb-client:2.0.3 (1 constraints: 870d902f)
+org.apache.kerby:kerb-common:2.0.3 (3 constraints: df2506b3)
+org.apache.kerby:kerb-identity:2.0.3 (1 constraints: 620cbd02)
+org.apache.kerby:kerb-server:2.0.3 (1 constraints: d40b6cf2)
+org.apache.kerby:kerb-simplekdc:2.0.3 (1 constraints: df0d853e)
 org.apache.tomcat.embed:tomcat-embed-el:9.0.76 (1 constraints: d41558cf)
 org.bouncycastle:bcpkix-jdk18on:1.78.1 (1 constraints: 43055a3b)
 org.bouncycastle:bcprov-jdk18on:1.78.1 (2 constraints: 851b50bd)

diff --git a/versions.props b/versions.props
@@ -41,7 +41,7 @@ org.apache.commons:commons-lang3=3.15.0
 org.apache.commons:commons-math3=3.6.1
 org.apache.curator:*=5.7.0
 org.apache.hadoop.thirdparty:*=1.2.0
-org.apache.hadoop:*=3.3.6
+org.apache.hadoop:*=3.4.0
 org.apache.httpcomponents:httpclient=4.5.14
 org.apache.httpcomponents:httpcore=4.4.16
 org.apache.httpcomponents:httpmime=4.5.14