Merge branch 'main' into wait-for-state-variant

.github/workflows/bin-solr-test.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/docker-test.yml

@@ -21,7 +21,7 @@ jobs:
     env:
       SOLR_DOCKER_IMAGE_REPO: github-pr/solr
       SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}}
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/gradle-precommit.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/solrj-test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

dev-docs/FAQ.adoc

@@ -95,7 +95,7 @@ If you don't yet have an account, you have to ask for one in the 'users' or 'dev
 === Where can I find information about test history?
 
 * http://fucit.org/solr-jenkins-reports/failure-report.html
-* https://ge.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr*
+* https://develocity.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr*
 * https://lists.apache.org[Solr mailing list archives especially builds]
 
 === How can I build the JavaDoc's and the Reference Guide?

dev-docs/v2-api-conventions.adoc

@@ -1,3 +1,4 @@
+= API Design
 == HTTP Paths
 
 Where possible, each v2 API is given an HTTP path that reflects the resource type and/or name most relevant to its functionality.
@@ -69,8 +70,8 @@ For use within the v2 API, the four "popular" HTTP methods have the following se
 == Errors
 
 v2 APIs should be consistent in how they report errors.  Throwing a `SolrException` will convey
-1.the error code as the HTTP response status code, as `responseHeader.status` and as `error.code`, and
-1.the error message as `error.msg`.
+1. The error code as the HTTP response status code, as `responseHeader.status` and as `error.code`, and
+2. The error message as `error.msg`.
 
 API calls that reference a specific resource (e.g. `specificCollName`, `specificAliasName`, `specificPropertyName` and others per the above list) that do not exist should return `SolrException.ErrorCode.NOT_FOUND` (HTTP 404).
 
@@ -82,3 +83,23 @@ Often these operations were initially conceived of as procedural "commands" and
 
 Solr's v2 API currently accommodates these "command" APIs by appending the command name (often a verb like "unload", "reload", or "split") onto the otherwise "resource"-based path.
 For example: Solr's core "unload" command uses the API `POST /api/cores/specificCoreName/unload`.
+
+= JAX-RS Implementation Conventions
+
+== Streaming
+
+Solr has a number of APIs that return binary file data or other arbitrary content, such as the "replication" APIs used to pull index files from other cores.
+Please use the following conventions when implementing similar endpoints:
+1. `@Operation` annotations use an "extension property" to indicate to codegen tools that the API output is "raw" or untyped.  For example:
++
+```
+  @Operation(
+      summary = "Return the data stored in a specified ZooKeeper node",
+      tags = {"zookeeper-read"},
+      extensions = {
+        @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})
+      })
+```
+2. Interface methods should return a type that implements the JAX-RS `StreamingOutput` interface.
+
+See the `fetchFile()` method in `ReplicationApis.java` for a concrete example.

gradle/develocity.gradle

@@ -21,7 +21,7 @@ def isCIBuild = System.getenv().keySet().any { it ==~ /(?i)((JENKINS|HUDSON)(_\w
 // https://docs.gradle.com/enterprise/gradle-plugin/
 
 develocity {
-    server = "https://ge.apache.org"
+    server = "https://develocity.apache.org"
     projectId = "solr"
 
     buildScan {

gradle/template.gradle.properties

@@ -73,7 +73,7 @@ systemProp.file.encoding=UTF-8
 # Set up gradle JVM defaults.
 #
 # We also open up internal compiler modules for spotless/ google java format.
-org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \
+org.gradle.jvmargs=-Xmx1g -XX:ReservedCodeCacheSize=256m -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \
  --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
  --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
  --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \

gradle/testing/failed-tests-at-end.gradle

@@ -22,7 +22,7 @@ def failedTests = new LinkedHashSet() // for dedupe due to weird afterTest class
 def genFailInfo(def task, TestDescriptor desc) {
   boolean isSuite = (desc.name == 'classMethod')
   def name = isSuite ? desc.className : "${desc.className}.${desc.name}"
-  def historyUrl = "https://ge.apache.org/scans/tests?search.rootProjectNames=solr-root&tests.container=$desc.className"
+  def historyUrl = "https://develocity.apache.org/scans/tests?search.rootProjectNames=solr-root&tests.container=$desc.className"
   if (!isSuite) { // is test method specific
     historyUrl += "&tests.test=$desc.name"
     historyUrl += " http://fucit.org/solr-jenkins-reports/history-trend-of-recent-failures.html#series/$name"

settings.gradle

@@ -25,7 +25,7 @@ pluginManagement {
 }
 
 plugins {
-    id 'com.gradle.develocity' version '3.18.1'
+    id 'com.gradle.develocity' version '3.18.2'
     id 'com.gradle.common-custom-user-data-gradle-plugin' version '2.0.2'
 }
 

solr/CHANGES.txt

@@ -169,6 +169,10 @@ Bug Fixes
 * SOLR-17587: Metrics: wt=prometheus fix for non-compliant exposition format containing duplicate TYPE lines.
   (Matthew Biscocho via David Smiley)
 
+* SOLR-17221: If multiple query param keys are sent that only vary by case, they were wrongly merged when
+  doing distributed search (sharded collections). This could likely occur for fielded parameters such as
+  f.CASE_SENSITIVE_FIELD.facet.limit=50. (Yue Yu, David Smiley)
+
 Dependency Upgrades
 ---------------------
 * SOLR-17471: Upgrade Lucene to 9.12.1. (Pierre Salagnac, Christine Poerschke)
@@ -187,6 +191,9 @@ Other Changes
 
 * SOLR-17589: Prevent error log entry on solr server due to initial HEAD request from HttpJdkSolrClient. (Paul Blanchaert via James Dyer)
 
+* SOLR-17611: SolrJ's User-Agent header now uses the version of SolrJ.  There's a corresponding
+  HttpSolrCall.getUserAgentSolrVersion to parse it.  (David Smiley)
+
 * SOLR-17581: Introduce new test variant of waitForState(), that does not wait on live node changes when we're only
  interested in the collection state. (Pierre Salagnac)
 

solr/api/build.gradle

@@ -55,6 +55,8 @@ resolve {
   outputDir = file(project.openApiSpecDir)
   outputFileName = "solr-openapi-${version}"
   prettyPrint = true
+// Ignore resources not annotated with 'Operation', useful for omitting endpoints from OAS
+  readAllResources = false
 }
 
 dependencies {

solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java

@@ -89,6 +89,7 @@ SolrJerseyResponse uploadConfigSet(
 
     @PUT
     @Path("{filePath:.+}")
+    @Operation(summary = "Create a new configset.", tags = "configsets")
     SolrJerseyResponse uploadConfigSetFile(
         @PathParam("configSetName") String configSetName,
         @PathParam("filePath") String filePath,

solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationApis.java

@@ -16,7 +16,7 @@
  */
 package org.apache.solr.client.api.endpoint;
 
-import static org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;
+import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;
 
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -59,10 +59,9 @@ FileListResponse fetchFileList(
   @CoreApiParameters
   @Operation(
       summary = "Get a stream of a specific file path of a core",
-      tags = {"core-replication"},
-      extensions = { // TODO Remove as a part of SOLR-17562
-        @Extension(
-            properties = {@ExtensionProperty(name = OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
+      tags = {"replication"},
+      extensions = {
+        @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})
       })
   @Path("/files/{filePath}")
   StreamingOutput fetchFile(

solr/api/src/java/org/apache/solr/client/api/endpoint/ZooKeeperReadApis.java

@@ -16,15 +16,19 @@
  */
 package org.apache.solr.client.api.endpoint;
 
+import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;
+
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.extensions.Extension;
+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.core.MediaType;
-import org.apache.solr.client.api.model.ZooKeeperFileResponse;
+import jakarta.ws.rs.core.StreamingOutput;
 import org.apache.solr.client.api.model.ZooKeeperListChildrenResponse;
 
 /** V2 API definitions for Solr's ZooKeeper ready-proxy endpoint */
@@ -35,9 +39,12 @@ public interface ZooKeeperReadApis {
   @Path("/data{zkPath:.+}")
   @Operation(
       summary = "Return the data stored in a specified ZooKeeper node",
-      tags = {"zookeeper-read"})
+      tags = {"zookeeper-read"},
+      extensions = {
+        @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})
+      })
   @Produces({"application/vnd.apache.solr.raw", MediaType.APPLICATION_JSON})
-  ZooKeeperFileResponse readNode(
+  StreamingOutput readNode(
       @Parameter(description = "The path of the node to read from ZooKeeper") @PathParam("zkPath")
           String zkPath);
 
@@ -48,7 +55,7 @@ ZooKeeperFileResponse readNode(
   @GET
   @Path("/data/security.json")
   @Produces({"application/vnd.apache.solr.raw", MediaType.APPLICATION_JSON})
-  ZooKeeperFileResponse readSecurityJsonNode();
+  StreamingOutput readSecurityJsonNode();
 
   @GET
   @Path("/children{zkPath:.*}")

solr/api/src/java/org/apache/solr/client/api/util/Constants.java

@@ -29,7 +29,12 @@ private Constants() {
 
   public static final String CORE_NAME_PATH_PARAMETER = "coreName";
 
+  // Annotation used on endpoints that should be skipped by code-generation
   public static final String OMIT_FROM_CODEGEN_PROPERTY = "omitFromCodegen";
+  // Annotation used to indicate that the specified API can return arbitrary, unparseable content
+  // such as ZK or filestore files
+  public static final String RAW_OUTPUT_PROPERTY = "rawOutput";
+
   public static final String GENERIC_ENTITY_PROPERTY = "genericEntity";
 
   public static final String BINARY_CONTENT_TYPE_V2 = "application/vnd.apache.solr.javabin";

solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java

@@ -308,10 +308,7 @@ public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOExcepti
 
     if (responseParser instanceof InputStreamResponseParser) {
       // SPECIAL CASE
-      NamedList<Object> namedList = new NamedList<>();
-      namedList.add("stream", byteBuffer.toInputStream());
-      namedList.add("responseStatus", 200); // always by this point
-      return namedList;
+      return InputStreamResponseParser.createInputStreamNamedList(200, byteBuffer.toInputStream());
     }
 
     // note: don't bother using the Reader variant; it often throws UnsupportedOperationException

solr/core/src/java/org/apache/solr/handler/admin/ZookeeperRead.java

@@ -21,19 +21,21 @@
 import static org.apache.solr.security.PermissionNameProvider.Name.ZK_READ_PERM;
 
 import jakarta.inject.Inject;
+import jakarta.ws.rs.WebApplicationException;
+import jakarta.ws.rs.core.StreamingOutput;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.solr.client.api.endpoint.ZooKeeperReadApis;
-import org.apache.solr.client.api.model.ZooKeeperFileResponse;
 import org.apache.solr.client.api.model.ZooKeeperListChildrenResponse;
 import org.apache.solr.client.api.model.ZooKeeperStat;
 import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.client.solrj.impl.XMLResponseParser;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.handler.admin.api.AdminAPIBase;
 import org.apache.solr.jersey.PermissionName;
@@ -64,15 +66,15 @@ public ZookeeperRead(CoreContainer coreContainer, SolrQueryRequest req, SolrQuer
   /** Request contents of a znode, except security.json */
   @Override
   @PermissionName(ZK_READ_PERM)
-  public ZooKeeperFileResponse readNode(String zkPath) {
+  public StreamingOutput readNode(String zkPath) {
     zkPath = sanitizeZkPath(zkPath);
     return readNodeAndAddToResponse(zkPath);
   }
 
   /** Request contents of the security.json node */
   @Override
   @PermissionName(SECURITY_READ_PERM)
-  public ZooKeeperFileResponse readSecurityJsonNode() {
+  public StreamingOutput readSecurityJsonNode() {
     return readNodeAndAddToResponse("/security.json");
   }
 
@@ -141,18 +143,19 @@ private String guessMime(byte firstByte) {
   }
 
   /** Reads content of a znode */
-  private ZooKeeperFileResponse readNodeAndAddToResponse(String zkPath) {
-    final ZooKeeperFileResponse zkFileResponse =
-        instantiateJerseyResponse(ZooKeeperFileResponse.class);
-
+  private StreamingOutput readNodeAndAddToResponse(String zkPath) {
     byte[] d = readPathFromZookeeper(zkPath);
     if (d == null || d.length == 0) {
-      zkFileResponse.zkData = EMPTY;
-      return zkFileResponse;
+      d = new byte[0];
     }
 
-    zkFileResponse.output = new ContentStreamBase.ByteArrayStream(d, null, guessMime(d[0]));
-    return zkFileResponse;
+    final var bytesToWrite = d;
+    return new StreamingOutput() {
+      @Override
+      public void write(OutputStream output) throws IOException, WebApplicationException {
+        output.write(bytesToWrite);
+      }
+    };
   }
 
   /** Reads a single node from zookeeper and return as byte array */

solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java

@@ -74,6 +74,7 @@
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.solr.api.ApiBag;
 import org.apache.solr.api.V2HttpCall;
+import org.apache.solr.client.api.util.SolrVersion;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.common.SolrException;
@@ -1170,6 +1171,24 @@ protected Object _getHandler() {
     return handler;
   }
 
+  /**
+   * Gets the client (user-agent) SolrJ version, or null if isn't SolrJ. Note that older SolrJ
+   * clients prior to 9.9 present themselves as 1.0 or 2.0.
+   */
+  public SolrVersion getUserAgentSolrVersion() {
+    String header = req.getHeader("User-Agent");
+    if (header == null || !header.startsWith("Solr")) {
+      return null;
+    }
+    try {
+      return SolrVersion.valueOf(header.substring(header.lastIndexOf(' ') + 1));
+    } catch (Exception e) {
+      // unexpected but let's not freak out
+      assert false : e.toString();
+      return null;
+    }
+  }
+
   private AuthorizationContext getAuthCtx() {
 
     String resource = getPath();