Merge branch 'main' into fix_missing_auth_listeners

.github/workflows/gradle-precommit.yml

@@ -42,6 +42,6 @@ jobs:
           ${{ runner.os }}-gradle-
 
     - name: Run gradle check (without tests)
-      run: ./gradlew check -x test -Ptask.times=true
+      run: ./gradlew check -x test -Ptask.times=true --continue
 
     - uses: gradle/wrapper-validation-action@v3

solr/CHANGES.txt

@@ -150,6 +150,8 @@ Other Changes
 
 * SOLR-17675: Remove DirContext param from DirectoryFactory.create method. (Bruno Roustant, David Smiley)
 
+* SOLR-16903: Switch CoreContainer#getSolrHome to return Path instead of String (Andrey Bozhko)
+
 ==================  9.9.0 ==================
 New Features
 ---------------------
@@ -182,6 +184,8 @@ Optimizations
 ---------------------
 * SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)
 
+* SOLR-17669: Reduced memory usage in SolrJ getBeans() method when handling dynamic fields with wildcards. (Martin Anzinger)
+
 Bug Fixes
 ---------------------
 * SOLR-17587: Metrics: wt=prometheus fix for non-compliant exposition format containing duplicate TYPE lines.
@@ -248,6 +252,8 @@ Other Changes
 
 * SOLR-17670: Fix unnecessary memory allocation caused by a large reRankDocs param. (JiaBao Gao)
 
+* SOLR-17518: Deprecate UpdateRequest.getXml() and replace it with XMLRequestWriter. (Pierre Salagnac)
+
 ==================  9.8.0 ==================
 New Features
 ---------------------

solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java

@@ -38,6 +38,7 @@
 import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
 import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
+import org.apache.solr.client.solrj.impl.XMLRequestWriter;
 import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
 import org.apache.solr.client.solrj.request.RequestWriter;
 import org.apache.solr.common.SolrDocument;
@@ -78,8 +79,8 @@ public class EmbeddedSolrServer extends SolrClient {
 
   @SuppressWarnings("ImmutableEnumChecker")
   public enum RequestWriterSupplier {
-    JavaBin(() -> new BinaryRequestWriter()),
-    XML(() -> new RequestWriter());
+    JavaBin(BinaryRequestWriter::new),
+    XML(XMLRequestWriter::new);
 
     private final Supplier<RequestWriter> supplier;
 

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -2411,9 +2411,8 @@ public boolean isLoaded(String name) {
   }
 
   /** The primary path of a Solr server's config, cores, and misc things. Absolute. */
-  // TODO return Path
-  public String getSolrHome() {
-    return solrHome.toString();
+  public Path getSolrHome() {
+    return solrHome;
   }
 
   /**

solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java

@@ -17,7 +17,6 @@
 
 package org.apache.solr.core;
 
-import java.nio.file.Path;
 import java.util.Map;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CoreAdminParams;
@@ -58,7 +57,7 @@ public static SyntheticSolrCore createAndRegisterCore(
     CoreDescriptor syntheticCoreDescriptor =
         new CoreDescriptor(
             syntheticCoreName,
-            Path.of(coreContainer.getSolrHome(), syntheticCoreName),
+            coreContainer.getSolrHome().resolve(syntheticCoreName),
             coreProps,
             coreContainer.getContainerProperties(),
             coreContainer.getZkController());

solr/core/src/java/org/apache/solr/core/ZkContainer.java

@@ -22,7 +22,6 @@
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Predicate;
@@ -87,11 +86,11 @@ public void initZooKeeper(final CoreContainer cc, CloudConfig config) {
     // TODO: remove after updating to an slf4j based zookeeper
     System.setProperty("zookeeper.jmx.log4j.disable", "true");
 
-    String solrHome = cc.getSolrHome();
+    Path solrHome = cc.getSolrHome();
     if (zkRun != null) {
       String zkDataHome =
-          System.getProperty("zkServerDataDir", Paths.get(solrHome).resolve("zoo_data").toString());
-      String zkConfHome = System.getProperty("zkServerConfDir", solrHome);
+          System.getProperty("zkServerDataDir", solrHome.resolve("zoo_data").toString());
+      String zkConfHome = System.getProperty("zkServerConfDir", solrHome.toString());
       zkServer =
           new SolrZkServer(
               stripChroot(zkRun),

solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java

@@ -31,7 +31,6 @@
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Date;
@@ -76,7 +75,7 @@ public class DistribFileStore implements FileStore {
 
   public DistribFileStore(CoreContainer coreContainer) {
     this.coreContainer = coreContainer;
-    this.solrHome = Paths.get(this.coreContainer.getSolrHome());
+    this.solrHome = this.coreContainer.getSolrHome();
   }
 
   @Override

solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerLocal.java

@@ -23,7 +23,6 @@
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Collections;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.CommandOperation;
@@ -40,7 +39,7 @@ public class SecurityConfHandlerLocal extends SecurityConfHandler {
 
   public SecurityConfHandlerLocal(CoreContainer coreContainer) {
     super(coreContainer);
-    securityJsonPath = Paths.get(coreContainer.getSolrHome()).resolve("security.json");
+    securityJsonPath = coreContainer.getSolrHome().resolve("security.json");
   }
 
   /**

solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java

@@ -146,7 +146,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw
     }
     if (cc != null) {
       rsp.add("solr_home", cc.getSolrHome());
-      rsp.add("core_root", cc.getCoreRootDirectory().toString());
+      rsp.add("core_root", cc.getCoreRootDirectory());
     }
 
     rsp.add("lucene", getLuceneInfo());

solr/core/src/java/org/apache/solr/handler/admin/api/CoreStatus.java

@@ -48,6 +48,8 @@
  */
 public class CoreStatus extends CoreAdminAPIBase implements CoreApis.GetStatus {
 
+  private static boolean INDEX_INFO_DEFAULT_VALUE = true;
+
   @Inject
   public CoreStatus(
       CoreContainer coreContainer,
@@ -60,13 +62,15 @@ public CoreStatus(
   @Override
   @PermissionName(CORE_READ_PERM)
   public CoreStatusResponse getAllCoreStatus(Boolean indexInfo) throws IOException {
-    return fetchStatusInfo(coreContainer, null, indexInfo);
+    final var indexInfoNeeded = indexInfo == null ? INDEX_INFO_DEFAULT_VALUE : indexInfo;
+    return fetchStatusInfo(coreContainer, null, indexInfoNeeded);
   }
 
   @Override
   @PermissionName(CORE_READ_PERM)
   public CoreStatusResponse getCoreStatus(String coreName, Boolean indexInfo) throws IOException {
-    return fetchStatusInfo(coreContainer, coreName, indexInfo);
+    final var indexInfoNeeded = indexInfo == null ? INDEX_INFO_DEFAULT_VALUE : indexInfo;
+    return fetchStatusInfo(coreContainer, coreName, indexInfoNeeded);
   }
 
   public static CoreStatusResponse fetchStatusInfo(

solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java

@@ -24,7 +24,6 @@
 import java.io.InputStream;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -286,7 +285,7 @@ public void writeMap(EntryWriter ew) throws IOException {
             new PackageResourceLoader(
                 "PACKAGE_LOADER: " + parent.name() + ":" + version,
                 paths,
-                Paths.get(coreContainer.getSolrHome()),
+                coreContainer.getSolrHome(),
                 coreContainer.getResourceLoader().getClassLoader());
       }
 

solr/core/src/java/org/apache/solr/search/join/HashRangeQuery.java

@@ -38,9 +38,14 @@
 import org.apache.solr.common.util.Hash;
 import org.apache.solr.search.SolrCache;
 import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.SolrSearcherRequirer;
 
-public class HashRangeQuery extends Query implements SolrSearcherRequirer {
+/**
+ * Matches documents where the specified field hashes to a value within the given range. Can be used
+ * to create a filter that will only match documents falling within a certain shard's hash range.
+ *
+ * @see HashRangeQParser
+ */
+public class HashRangeQuery extends Query {
 
   protected final String field;
   protected final int lower;
@@ -91,6 +96,9 @@ private int[] getCache(LeafReaderContext context) throws IOException {
         if (cacheHelper == null) {
           return null;
         }
+        if (!(searcher instanceof SolrIndexSearcher)) { // e.g. delete-by-query
+          return null;
+        }
         @SuppressWarnings("unchecked")
         final SolrCache<IndexReader.CacheKey, int[]> cache =
             ((SolrIndexSearcher) searcher).getCache(CACHE_KEY_PREFIX + field);

solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java

@@ -270,8 +270,7 @@ public void testDBQWithUnsupportedQueryReturns400() throws Exception {
     final var unsupportedQueryExamples =
         new String[] {
           "{!join from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}",
-          "{!graph from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}",
-          "{!hash_range f=\"foo_i\" l=\"0\" u=\"12345\"}"
+          "{!graph from=expected_shard_s to=expected_shard_s v=\"expected_shard_s:5\"}"
         };
 
     update(params()).add(doc(f("id", UUID.randomUUID().toString()))).process(COLLECTION_CLIENT);

solr/core/src/test/org/apache/solr/core/TestConfigSets.java

@@ -77,7 +77,7 @@ public void testConfigSetServiceFindsConfigSets() {
     CoreContainer container = null;
     try {
       container = setupContainer(TEST_PATH().resolve("configsets").toString());
-      Path solrHome = Paths.get(container.getSolrHome());
+      Path solrHome = container.getSolrHome();
 
       SolrCore core1 = container.create("core1", Map.of("configSet", "configset-2"));
       assertThat(core1.getCoreDescriptor().getName(), is("core1"));

solr/core/src/test/org/apache/solr/core/TestCoreContainer.java

@@ -95,7 +95,7 @@ public void testSolrHomeAndResourceLoader() throws Exception {
     try {
 
       // instance dir & resource loader for the CC
-      assertEquals(realSolrHome.toString(), cc.getSolrHome());
+      assertEquals(realSolrHome, cc.getSolrHome());
       assertEquals(realSolrHome, cc.getResourceLoader().getInstancePath());
 
     } finally {
@@ -983,7 +983,7 @@ public void testCoreInitFailuresOnReload() throws Exception {
 
     // -----
     // "fix" the bad collection
-    Path confDir = Path.of(cc.getSolrHome(), "col_bad", "conf");
+    Path confDir = cc.getSolrHome().resolve("col_bad").resolve("conf");
     Files.createDirectories(confDir);
     Files.copy(
         getFile("solr/collection1/conf/solrconfig-defaults.xml"),
@@ -1059,7 +1059,7 @@ public void testCoreInitFailuresOnReload() throws Exception {
     final long col_bad_old_start = getCoreStartTime(cc, "col_bad");
 
     Files.writeString(
-        Path.of(cc.getSolrHome(), "col_bad", "conf", "solrconfig.xml"),
+        cc.getSolrHome().resolve("col_bad").resolve("conf").resolve("solrconfig.xml"),
         "This is giberish, not valid XML <",
         StandardCharsets.UTF_8);
 

solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java

@@ -25,7 +25,6 @@
 import java.io.InputStream;
 import java.lang.invoke.MethodHandles;
 import java.nio.ByteBuffer;
-import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -320,7 +319,7 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu
       throws Exception {
     JettySolrRunner jetty = cluster.getRandomJetty(random());
     try (HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
-      PackageUtils.uploadKey(bytes, path, Paths.get(jetty.getCoreContainer().getSolrHome()));
+      PackageUtils.uploadKey(bytes, path, jetty.getCoreContainer().getSolrHome());
 
       final var syncReq = new FileStoreApi.SyncFile(path);
       final var syncRsp = syncReq.process(client);

solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java

@@ -52,6 +52,7 @@
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CoresApi;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -63,7 +64,6 @@
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.CachingDirectoryFactory;
@@ -1676,18 +1676,15 @@ private Date watchCoreStartAt(JettySolrRunner jettySolrRunner, final Date min)
     long timeSlept = 0;
 
     try (SolrClient adminClient = adminClient(jettySolrRunner)) {
-      SolrParams p = params("action", "status", "core", "collection1");
+      final var statusRequest = new CoresApi.GetCoreStatus("collection1");
       while (timeSlept < TIMEOUT) {
-        QueryRequest req = new QueryRequest(p);
-        req.setPath("/admin/cores");
         try {
-          NamedList<Object> data = adminClient.request(req);
-          for (String k : new String[] {"status", "collection1"}) {
-            Object o = data.get(k);
-            assertNotNull("core status rsp missing key: " + k, o);
-            data = (NamedList<Object>) o;
-          }
-          Date startTime = (Date) data.get("startTime");
+          final var statusResponse = statusRequest.process(adminClient).getParsed();
+          assertNotNull(statusResponse.status);
+          assertTrue(statusResponse.status.containsKey("collection1"));
+          final var coreStatus = statusResponse.status.get("collection1");
+          Date startTime = coreStatus.startTime;
+
           assertNotNull("core has null startTime", startTime);
           if (null == min || startTime.after(min)) {
             return startTime;

solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java

@@ -220,7 +220,7 @@ public void testCoreContainerMetrics() {
     assertTrue(metrics.containsKey("CONTAINER.version.specification"));
     assertTrue(metrics.containsKey("CONTAINER.version.implementation"));
     Gauge<?> g = (Gauge<?>) metrics.get("CONTAINER.fs.path");
-    assertEquals(g.getValue(), cc.getSolrHome());
+    assertEquals(g.getValue(), cc.getSolrHome().toString());
   }
 
   @Test

solr/core/src/test/org/apache/solr/search/SolrSearcherRequirementDetectorTest.java

@@ -71,7 +71,7 @@ public void testDetectsWhenQueriesDoRequireSolrSearcher_TopLevel() {
   }
 
   @Test
-  public void testDeteectsWhenQueriesDoRequireSolrSearcher_Nested() {
+  public void testDetectsWhenQueriesRequireSolrSearcher_Nested() {
     final var termQuery = new TermQuery(new Term("someField", "someFieldValue"));
     final var termQuery2 = new TermQuery(new Term("someField", "someOtherFieldValue"));
     final var joinQuery = new JoinQuery("fromField", "toField", "someCoreName", termQuery);

solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java

@@ -361,11 +361,17 @@ public void testNRTRestart() throws Exception {
         // TODO: could separate this out into a different test method, but this should suffice for
         // now
         pullJetty.start(true);
+        waitForState(
+            "Pull jetty replicas didn't become active in time",
+            COLL,
+            ((liveNodes, collectionState) ->
+                collectionState.getReplicas(pullJettyF.getNodeName()).stream()
+                    .allMatch(rep -> rep.getState() == Replica.State.ACTIVE)));
         AtomicBoolean done = new AtomicBoolean();
         long runMinutes = 1;
         long finishTimeMs =
             new Date().getTime() + TimeUnit.MILLISECONDS.convert(runMinutes, TimeUnit.MINUTES);
-        JettySolrRunner[] jettys = new JettySolrRunner[] {nrtJettyF, pullJettyF};
+        JettySolrRunner[] jettys = new JettySolrRunner[] {pullJettyF, nrtJettyF};
         Random threadRandom = new Random(r.nextInt());
         Future<Integer> f =
             executor.submit(
@@ -386,6 +392,12 @@ public void testNRTRestart() throws Exception {
                       log.info("restarting {} ...", idx);
                       toManipulate.start(true);
                       log.info("restarted {}.", idx);
+                      waitForState(
+                          toManipulate.getNodeName() + " replicas didn't become active in time",
+                          COLL,
+                          ((liveNodes, collectionState) ->
+                              collectionState.getReplicas(toManipulate.getNodeName()).stream()
+                                  .allMatch(rep -> rep.getState() == Replica.State.ACTIVE)));
                     } catch (Exception e) {
                       throw new RuntimeException(e);
                     }