Merge remote-tracking branch 'apache/main'

build.gradle

@@ -21,7 +21,7 @@ import java.time.format.DateTimeFormatter
 plugins {
   id "base"
   id "com.palantir.consistent-versions" version "2.11.0"
-  id "org.owasp.dependencycheck" version "7.2.0"
+  id "org.owasp.dependencycheck" version "7.4.1"
   id 'ca.cutterslade.analyze' version "1.9.0"
   id 'de.thetaphi.forbiddenapis' version '3.4' apply false
   id "de.undercouch.download" version "5.2.0" apply false

dev-docs/lucene-upgrade.md

@@ -44,6 +44,8 @@ gradlew --write-locks
 
 ```
 gradlew updateLicenses
+
+git add solr/licenses
 ```
 
 ## Code

gradle/validation/owasp-dependency-check.gradle

@@ -25,9 +25,9 @@ def resources = scriptResources(buildscript)
 configure(rootProject) {
   dependencyCheck {
     failBuildOnCVSS = propertyOrDefault("validation.owasp.threshold", 7) as Integer
-    formats = ['HTML', 'JSON']
+    formats = ['ALL']
     skipProjects = [':solr:solr-ref-guide']
-    skipConfigurations = ['unifiedClasspath']
+    skipConfigurations = ['unifiedClasspath', 'permitUnusedDeclared']
     suppressionFile = file("${resources}/exclusions.xml")
   }
 

gradle/validation/owasp-dependency-check/exclusions.xml

@@ -51,4 +51,25 @@
     <packageUrl regex="true">^pkg:maven/org\.carrot2\.shaded/carrot2\-guava@.*$</packageUrl>
     <cpe>cpe:/a:google:guava</cpe>
   </suppress>
+  <suppress>
+    <notes><![CDATA[Apache Calcite Avatica has separate releases from Apache Calcite]]></notes>
+    <packageUrl regex="true">^pkg:maven/org\.apache\.calcite\.avatica/.*@.*$</packageUrl>
+    <cpe>cpe:/a:apache:calcite</cpe>
+  </suppress>
+  <suppress>
+    <notes><![CDATA[jetty-servlet-api has separate releases from Jetty itself]]></notes>
+    <packageUrl regex="true">^pkg:maven/org\.eclipse\.jetty\.toolchain/jetty\-servlet\-api@.*$</packageUrl>
+    <cpe>cpe:/a:eclipse:jetty</cpe>
+    <cpe>cpe:/a:jetty:jetty</cpe>
+  </suppress>
+  <suppress>
+    <notes><![CDATA[hadoop-shaded-guava-1.1.1.jar is not Apache Hadoop 1.1.1]]></notes>
+    <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop\.thirdparty/hadoop\-shaded\-guava@.*$</packageUrl>
+    <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
+  </suppress>
+  <suppress>
+    <notes><![CDATA[hadoop-client-runtime-3.*.jar is not Apache Hadoop 1.1.1]]></notes>
+    <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop/hadoop\-client\-runtime@3.*$</packageUrl>
+    <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
+  </suppress>
 </suppressions>

solr/CHANGES.txt

@@ -81,13 +81,20 @@ Improvements
   (Houston Putman)
 
 * SOLR-16565: posting the same file to the package store should not throw an error (noble)
+
+* SOLR-8975: Introduce Builder setters for setters that exist on SolrClients for responseParser, requestWriter.
+  Deprecated these SolrClient setters useMultiPartPost, and followRedirects. (Eric Pugh, David Smiley, Kevin Risden)
+
 Optimizations
 ---------------------
 
 * SOLR-16515: Remove synchronized access to cachedOrdMaps in SlowCompositeReaderWrapper (Dennis Berger, Torsten Bøgh Köster, Marco Petris)
 
 * SOLR-16555: SolrIndexSearcher - FilterCache intersections/andNot should not clone bitsets repeatedly (Kevin Risden, David Smiley)
 
+* SOLR-15732: queries to missing collection are slow (noble)
+
+
 Bug Fixes
 ---------------------
 
@@ -116,6 +123,9 @@ Bug Fixes
 
 * SOLR-16165: Rare Deadlock in SlotAcc initialization (Justin Sweeney, noble)
 
+* SOLR-16473: Fix race condition in shard split when a sub-shard is put in recovery state. (Andy Vuong via Bruno Roustant)
+
+* SOLR-10458: Fix followRedirect property on HttpSolrClient not set when using Builder pattern. (Eric Pugh)
 
 Build
 ---------------------
@@ -125,10 +135,14 @@ Build
 
 * SOLR-16476: Remove commons-text dependency from solr-core (Kevin Risden)
 
-* SOLR-16442: Upgrade to Lucene 9.4.1 (Christine Poerschke)
+* SOLR-16442: Upgrade to Lucene 9.4.2 (Christine Poerschke)
 
 * SOLR-16508: Upgrade gradle wrapper to 7.6 (Kevin Risden)
 
+* SOLR-16578: Upgrade to errorprone 2.16 (Kevin Risden)
+
+* SOLR-16581: Upgrade OWASP dependency check to 7.4.1 (Kevin Risden)
+
 Other Changes
 ---------------------
 * SOLR-16545: Upgrade Carrot2 to 4.5.0 (Dawid Weiss)
@@ -150,6 +164,16 @@ Other Changes
 
 * SOLR-15861: ConcurrentUpdateSolrClient should work with ManagedExecutorService (Sammy Chu, Kevin Risden)
 
+* SOLR-16575: splitshard should honour createNodeSet (noble)
+
+* SOLR-16568: Upgrade woodstox-core to 6.4.0 (Kira Traynor)
+
+* SOLR-16579: Upgrade Jackson to 2.14.1 (Kevin Risden)
+
+* SOLR-16562: Upgrade to Caffeine 3.1.2 (Kevin Risden)
+
+* SOLR-16569: Add java system property to overseer queue size (Nick Ginther via noble)
+
 ==================  9.1.0 ==================
 
 New Features

solr/core/src/java/org/apache/solr/cloud/Overseer.java

@@ -152,7 +152,8 @@ public class Overseer implements SolrCloseable {
   public static final int STATE_UPDATE_DELAY = ZkStateReader.STATE_UPDATE_DELAY;
   public static final int STATE_UPDATE_BATCH_SIZE =
       Integer.getInteger("solr.OverseerStateUpdateBatchSize", 10000);
-  public static final int STATE_UPDATE_MAX_QUEUE = 20000;
+  public static final int STATE_UPDATE_MAX_QUEUE =
+      Integer.getInteger("solr.OverseerStateUpdateMaxQueueSize", 20000);
 
   public static final int NUM_RESPONSES_TO_STORE = 10000;
   public static final String OVERSEER_ELECT = "/overseer_elect";

solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java

@@ -18,6 +18,7 @@
 package org.apache.solr.cloud.api.collections;
 
 import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable.CORE_IDX;
+import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.RANDOM;
 import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
 import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
@@ -38,6 +39,7 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.solr.client.solrj.cloud.DistribStateManager;
@@ -576,7 +578,12 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList<O
               .assignNrtReplicas(numNrt.get())
               .assignTlogReplicas(numTlog.get())
               .assignPullReplicas(numPull.get())
-              .onNodes(new ArrayList<>(clusterState.getLiveNodes()))
+              .onNodes(
+                  Assign.getLiveOrLiveAndCreateNodeSetList(
+                      clusterState.getLiveNodes(),
+                      message,
+                      RANDOM,
+                      ccc.getSolrCloudManager().getDistribStateManager()))
               .build();
       Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(ccc.getCoreContainer());
       List<ReplicaPosition> replicaPositions =
@@ -779,6 +786,20 @@ public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList<O
         } else {
           ccc.offerStateUpdate(m);
         }
+        // Wait for the sub-shards to change to the RECOVERY state before creating the replica
+        // cores. Otherwise, there is a race condition and some recovery updates may be lost.
+        zkStateReader.waitForState(
+            collectionName,
+            60,
+            TimeUnit.SECONDS,
+            (collectionState) -> {
+              for (String subSlice : subSlices) {
+                if (!collectionState.getSlice(subSlice).getState().equals(Slice.State.RECOVERY)) {
+                  return false;
+                }
+              }
+              return true;
+            });
       }
 
       t = timings.sub("createCoresForReplicas");

solr/core/src/java/org/apache/solr/core/backup/BackupManager.java

@@ -252,7 +252,7 @@ public void uploadConfigDir(
       String sourceConfigName, String targetConfigName, ConfigSetService configSetService)
       throws IOException {
     URI source = repository.resolveDirectory(getZkStateDir(), CONFIG_STATE_DIR, sourceConfigName);
-    Preconditions.checkState(repository.exists(source), "Path {} does not exist", source);
+    Preconditions.checkState(repository.exists(source), "Path %s does not exist", source);
     uploadConfigToSolrCloud(configSetService, source, targetConfigName, "");
   }
 

solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java

@@ -47,6 +47,7 @@
 import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
 import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
 import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;
 import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
 import static org.apache.solr.common.params.CollectionAdminParams.PER_REPLICA_STATE;
 import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME;
@@ -933,7 +934,8 @@ public enum CollectionOperation implements CollectionOp {
                   NUM_SUB_SHARDS,
                   SPLIT_FUZZ,
                   SPLIT_BY_PREFIX,
-                  FOLLOW_ALIASES);
+                  FOLLOW_ALIASES,
+                  CREATE_NODE_SET_PARAM);
           return copyPropertiesWithPrefix(req.getParams(), map, PROPERTY_PREFIX);
         }),
     DELETESHARD_OP(

solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java

@@ -432,7 +432,7 @@ public void process(ResponseBuilder rb) throws IOException {
       newFilters.add(groupQuery);
     }
 
-    SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(null, newFilters);
+    SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(newFilters);
     if (pfilter.postFilter != null) {
       pfilter.postFilter.setLastDelegate(groupExpandCollector);
       collector = pfilter.postFilter;

solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java

@@ -288,7 +288,7 @@ public void init(PluginInfo info) {
             .maxConnectionsPerHost(maxConnectionsPerHost)
             .build();
     this.defaultClient.addListenerFactory(this.httpListenerFactory);
-    this.loadbalancer = new LBHttp2SolrClient(defaultClient);
+    this.loadbalancer = new LBHttp2SolrClient.Builder(defaultClient).build();
 
     initReplicaListTransformers(getParameter(args, "replicaRouting", null, sb));
 

solr/core/src/java/org/apache/solr/search/Grouping.java

@@ -309,8 +309,7 @@ public void execute() throws IOException {
     DocListAndSet out = new DocListAndSet();
     qr.setDocListAndSet(out);
 
-    SolrIndexSearcher.ProcessedFilter pf =
-        searcher.getProcessedFilter(cmd.getFilter(), cmd.getFilterList());
+    SolrIndexSearcher.ProcessedFilter pf = searcher.getProcessedFilter(cmd.getFilterList());
     final Query filterQuery = pf.filter;
     maxDoc = searcher.maxDoc();
 

solr/core/src/java/org/apache/solr/search/QueryCommand.java

@@ -31,7 +31,6 @@ public class QueryCommand {
   private String queryID;
   private boolean isQueryCancellable;
   private List<Query> filterList;
-  private DocSet filter;
   private Sort sort;
   private int offset;
   private int len;
@@ -79,10 +78,6 @@ public List<Query> getFilterList() {
    * @throws IllegalArgumentException if filter is not null.
    */
   public QueryCommand setFilterList(List<Query> filterList) {
-    if (filter != null) {
-      throw new IllegalArgumentException(
-          "Either filter or filterList may be set in the QueryCommand, but not both.");
-    }
     this.filterList = filterList;
     return this;
   }
@@ -93,10 +88,6 @@ public QueryCommand setFilterList(List<Query> filterList) {
    * @throws IllegalArgumentException if filter is not null.
    */
   public QueryCommand setFilterList(Query f) {
-    if (filter != null) {
-      throw new IllegalArgumentException(
-          "Either filter or filterList may be set in the QueryCommand, but not both.");
-    }
     filterList = null;
     if (f != null) {
       filterList = new ArrayList<>(2);
@@ -105,22 +96,6 @@ public QueryCommand setFilterList(Query f) {
     return this;
   }
 
-  public DocSet getFilter() {
-    return filter;
-  }
-
-  /**
-   * @throws IllegalArgumentException if filterList is not null.
-   */
-  public QueryCommand setFilter(DocSet filter) {
-    if (filterList != null) {
-      throw new IllegalArgumentException(
-          "Either filter or filterList may be set in the QueryCommand, but not both.");
-    }
-    this.filter = filter;
-    return this;
-  }
-
   public Sort getSort() {
     return sort;
   }