Merge main

.asf.yaml

@@ -29,6 +29,7 @@ github:
     branch_9_5: {}
     branch_9_6: {}
     branch_9_7: {}
+    branch_9_8: {}
     branch_9x: {}
 
   protected_tags:

.github/labeler.yml

@@ -123,11 +123,6 @@ module:gcs-repository:
       - any-glob-to-any-file:
           - solr/modules/gcs-repository/**
 
-module:hadoop-auth:
-  - changed-files:
-      - any-glob-to-any-file:
-          - solr/modules/hadoop-auth/**
-
 module:hdfs:
   - changed-files:
       - any-glob-to-any-file:

gradle/libs.versions.toml

@@ -38,7 +38,6 @@ apache-httpcomponents-httpclient = "4.5.14"
 apache-httpcomponents-httpcore = "4.4.16"
 apache-httpcomponents-httpmime = "4.5.14"
 apache-kafka = "3.7.1"
-apache-kerby = "2.0.3"
 apache-log4j = "2.21.0"
 apache-lucene = "9.11.1"
 apache-opennlp = "1.9.4"
@@ -229,14 +228,10 @@ apache-curator-client = { module = "org.apache.curator:curator-client", version.
 apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" }
 apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" }
 apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" }
-apache-hadoop-annotations = { module = "org.apache.hadoop:hadoop-annotations", version.ref = "apache-hadoop" }
-apache-hadoop-auth = { module = "org.apache.hadoop:hadoop-auth", version.ref = "apache-hadoop" }
 apache-hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "apache-hadoop" }
 apache-hadoop-client-minicluster = { module = "org.apache.hadoop:hadoop-client-minicluster", version.ref = "apache-hadoop" }
 apache-hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "apache-hadoop" }
-apache-hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "apache-hadoop" }
 apache-hadoop-hdfs = { module = "org.apache.hadoop:hadoop-hdfs", version.ref = "apache-hadoop" }
-apache-hadoop-minikdc = { module = "org.apache.hadoop:hadoop-minikdc", version.ref = "apache-hadoop" }
 apache-hadoop-thirdparty-shadedguava = { module = "org.apache.hadoop.thirdparty:hadoop-shaded-guava", version.ref = "apache-hadoop-thirdparty" }
 apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" }
 apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" }
@@ -245,8 +240,6 @@ apache-kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref
 apache-kafka-kafka213 = { module = "org.apache.kafka:kafka_2.13", version.ref = "apache-kafka" }
 apache-kafka-server-common = { module = "org.apache.kafka:kafka-server-common", version.ref = "apache-kafka" }
 apache-kafka-streams = { module = "org.apache.kafka:kafka-streams", version.ref = "apache-kafka" }
-apache-kerby-core = { module = "org.apache.kerby:kerb-core", version.ref = "apache-kerby" }
-apache-kerby-util = { module = "org.apache.kerby:kerb-util", version.ref = "apache-kerby" }
 apache-log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "apache-log4j" }
 apache-log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "apache-log4j" }
 apache-log4j-jul = { module = "org.apache.logging.log4j:log4j-jul", version.ref = "apache-log4j" }

gradle/testing/randomization/policies/solr-tests.policy

@@ -100,7 +100,7 @@ grant {
   permission java.lang.RuntimePermission "closeClassLoader";
   // needed by HttpSolrClient
   permission java.lang.RuntimePermission "getFileSystemAttributes";
-  // needed by hadoop auth (TODO: there is a cleaner way to handle this)
+  // needed by hadoop hdfs (TODO: there is a cleaner way to handle this)
   permission java.lang.RuntimePermission "loadLibrary.jaas";
   permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
   permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
@@ -135,17 +135,19 @@ grant {
   permission javax.management.MBeanServerPermission "findMBeanServer";
   permission javax.management.MBeanServerPermission "releaseMBeanServer";
   permission javax.management.MBeanTrustPermission "register";
-
-  // needed by hadoop auth
+  
+  // needed by hadoop hdfs
   permission javax.security.auth.AuthPermission "getSubject";
   permission javax.security.auth.AuthPermission "modifyPrincipals";
   permission javax.security.auth.AuthPermission "doAs";
-  permission javax.security.auth.AuthPermission "getLoginConfiguration";
-  permission javax.security.auth.AuthPermission "setLoginConfiguration";
   permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
   permission javax.security.auth.AuthPermission "modifyPublicCredentials";
   permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
 
+  // needed by crossdc
+  permission javax.security.auth.AuthPermission "getLoginConfiguration";
+  permission javax.security.auth.AuthPermission "setLoginConfiguration";
+
   // needed by hadoop security
   permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
   permission java.security.SecurityPermission "insertProvider";

gradle/validation/dependencies.gradle

@@ -179,9 +179,6 @@ allprojects {
                 handler.add(conf.name, libs.apache.httpcomponents.httpmime, {
                     because 'version alignment for consistency across project'
                 })
-                handler.add(conf.name, libs.apache.kerby.core, {
-                    because 'version alignment for consistency across project'
-                })
                 handler.add(conf.name, libs.apache.zookeeper.zookeeper, {
                     because 'version alignment for consistency across project'
                 })

gradle/validation/rat-sources.gradle

@@ -106,10 +106,6 @@ allprojects {
                     exclude "src/test-files/META-INF/services/*"
                     break
 
-                case ":solr:modules:hadoop-auth":
-                    exclude "src/test-files/**/*.conf"
-                    break
-
                 case ":solr:modules:hdfs":
                     exclude "src/test-files/**/*.aff"
                     exclude "src/test-files/**/*.dic"

settings.gradle

@@ -48,7 +48,6 @@ include "solr:modules:cross-dc"
 include "solr:modules:opentelemetry"
 include "solr:modules:extraction"
 include "solr:modules:gcs-repository"
-include "solr:modules:hadoop-auth"
 include "solr:modules:hdfs"
 include "solr:modules:jwt-auth"
 include "solr:modules:langid"

solr/CHANGES.txt

@@ -36,7 +36,8 @@ Improvements
 
 Optimizations
 ---------------------
-(No changes)
+* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.
+ (David Smiley)
 
 Bug Fixes
 ---------------------
@@ -93,6 +94,14 @@ Deprecation Removals
 
 * SOLR-17564: Remove code in Assign used for backwards compatibility with Collections created prior to 7.0 (Paul McArthur)
 
+* SOLR-17576: Remove deprecated master/slave option language from ReplicationHandler. (Eric Pugh)
+
+* SOLR-16781: Support for `<lib/>` directives (used in solrconfig.xml to add JARs on a core-by-core basis) has been removed.  Users
+  looking for similar functionality can use Solr's package manager.  Users that don't need to vary JAR access on a per-core basis
+  have many options, including the `<sharedLib/>` tag and directly modifying Solr's classpath prior to JVM startup. (Jason Gerlowski)
+
+* SOLR-17540: Removed the Hadoop Auth module, and thus Kerberos authentication and other exotic options. (Eric Pugh)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -130,6 +139,33 @@ Other Changes
 
 * SOLR-16903: Update CLI tools to use java.nio.file.Path instead of java.io.File (Andrey Bozhko)
 
+* SOLR-17568: SolrCloud no longer reroutes/proxies a core request to another node if not found locally. (David Smiley)
+
+==================  9.9.0 ==================
+New Features
+---------------------
+(No changes)
+
+Improvements
+---------------------
+(No changes)
+
+Optimizations
+---------------------
+(No changes)
+
+Bug Fixes
+---------------------
+(No changes)
+
+Dependency Upgrades
+---------------------
+(No changes)
+
+Other Changes
+---------------------
+(No changes)
+
 ==================  9.8.0 ==================
 New Features
 ---------------------
@@ -190,7 +226,9 @@ Optimizations
   that which consumes almost no memory, saving 1MB of memory per SolrCore.  (David Smiley)
 
 * SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,
-  reducing unnecessary data fetching. (Aparna Suresh, David Smiley)
+  reducing unnecessary data fetching.  Enhanced CloudSolrClient's HTTP ClusterStateProvider to use it, and to scale to
+  more collections better as well.
+   (Aparna Suresh, David Smiley)
 
 * SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)
 
@@ -229,6 +267,8 @@ Bug Fixes
 
 * SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid. (Jan Høydahl, Alexander Zagniotov)
 
+* SOLR-17574: Fix AllowListUrlChecker when liveNodes changes. Remove ClusterState.getHostAllowList (Bruno Roustant, David Smiley)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -258,6 +298,8 @@ led to the suppression of exceptions. (Andrey Bozhko)
 
 * SOLR-17556: "home" and "data" directories used by Solr examples have been updated to align with documented best practices. (Eric Pugh, Houston Putman)
 
+* SOLR-17577: Remove "solr.indexfetcher.sotimeout" system property that was for optimizing replication tests.  It was disabled, but not removed. (Eric Pugh)
+
 ==================  9.7.1 ==================
 Bug Fixes
 ---------------------

solr/bin/solr

@@ -318,13 +318,13 @@ fi
 if [ -z "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_OPTS:-}" ]; then
   echo "WARNING: SOLR_AUTHENTICATION_OPTS environment variable configured without associated SOLR_AUTH_TYPE variable"
   echo "         Please configure SOLR_AUTH_TYPE environment variable with the authentication type to be used."
-  echo "         Currently supported authentication types are [kerberos, basic]"
+  echo "         Currently supported authentication types are [basic]"
 fi
 
 if [ -n "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_CLIENT_BUILDER:-}" ]; then
   echo "WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE environment variables are configured together."
   echo "         Use SOLR_AUTH_TYPE environment variable to configure authentication type to be used. "
-  echo "         Currently supported authentication types are [kerberos, basic]"
+  echo "         Currently supported authentication types are [basic]"
   echo "         The value of SOLR_AUTHENTICATION_CLIENT_BUILDER environment variable will be ignored"
 fi
 
@@ -333,9 +333,6 @@ if [ -n "${SOLR_AUTH_TYPE:-}" ]; then
     basic)
       SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
       ;;
-    kerberos)
-      SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder"
-      ;;
     *)
       echo "ERROR: Value specified for SOLR_AUTH_TYPE environment variable is invalid."
       exit 1

solr/bin/solr.cmd

@@ -203,15 +203,15 @@ IF NOT DEFINED SOLR_AUTH_TYPE (
   IF DEFINED SOLR_AUTHENTICATION_OPTS (
     echo WARNING: SOLR_AUTHENTICATION_OPTS variable configured without associated SOLR_AUTH_TYPE variable
     echo          Please configure SOLR_AUTH_TYPE variable with the authentication type to be used.
-    echo          Currently supported authentication types are [kerberos, basic]
+    echo          Currently supported authentication types are [basic]
   )
 )
 
 IF DEFINED SOLR_AUTH_TYPE (
   IF DEFINED SOLR_AUTHENTICATION_CLIENT_BUILDER (
     echo WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE variables are configured together
     echo          Use SOLR_AUTH_TYPE variable to configure authentication type to be used
-    echo          Currently supported authentication types are [kerberos, basic]
+    echo          Currently supported authentication types are [basic]
     echo          The value of SOLR_AUTHENTICATION_CLIENT_BUILDER configuration variable will be ignored
   )
 )
@@ -220,12 +220,8 @@ IF DEFINED SOLR_AUTH_TYPE (
   IF /I "%SOLR_AUTH_TYPE%" == "basic" (
     set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
   ) ELSE (
-    IF /I "%SOLR_AUTH_TYPE%" == "kerberos" (
-      set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory"
-    ) ELSE (
-      echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid.
-      goto err
-    )
+    echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid.
+    goto err
   )
 )
 

solr/core/src/java/org/apache/solr/api/V2HttpCall.java

@@ -165,7 +165,7 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) {
           core = getCoreByCollection(collectionName, isPreferLeader);
           if (core == null) {
             // this collection exists , but this node does not have a replica for that collection
-            extractRemotePath(collectionName, collectionName);
+            extractRemotePath(collectionName);
             if (action == REMOTEQUERY) {
               action = ADMIN_OR_REMOTEQUERY;
               coreUrl = coreUrl.replace("/solr/", "/solr/____v2/c/");