Merge branch 'main' into feature/SOLR-17541

solr/CHANGES.txt

@@ -36,7 +36,8 @@ Improvements
 
 Optimizations
 ---------------------
-(No changes)
+* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.
+ (David Smiley)
 
 Bug Fixes
 ---------------------
@@ -95,6 +96,10 @@ Deprecation Removals
 
 * SOLR-17576: Remove deprecated master/slave option language from ReplicationHandler. (Eric Pugh)
 
+* SOLR-16781: Support for `<lib/>` directives (used in solrconfig.xml to add JARs on a core-by-core basis) has been removed.  Users
+  looking for similar functionality can use Solr's package manager.  Users that don't need to vary JAR access on a per-core basis
+  have many options, including the `<sharedLib/>` tag and directly modifying Solr's classpath prior to JVM startup. (Jason Gerlowski)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -132,6 +137,8 @@ Other Changes
 
 * SOLR-16903: Update CLI tools to use java.nio.file.Path instead of java.io.File (Andrey Bozhko)
 
+* SOLR-17568: SolrCloud no longer reroutes/proxies a core request to another node if not found locally. (David Smiley)
+
 ==================  9.8.0 ==================
 New Features
 ---------------------
@@ -231,6 +238,8 @@ Bug Fixes
 
 * SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid. (Jan Høydahl, Alexander Zagniotov)
 
+* SOLR-17574: Fix AllowListUrlChecker when liveNodes changes. Remove ClusterState.getHostAllowList (Bruno Roustant, David Smiley)
+
 Dependency Upgrades
 ---------------------
 (No changes)

solr/core/src/java/org/apache/solr/api/V2HttpCall.java

@@ -165,7 +165,7 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) {
           core = getCoreByCollection(collectionName, isPreferLeader);
           if (core == null) {
             // this collection exists , but this node does not have a replica for that collection
-            extractRemotePath(collectionName, collectionName);
+            extractRemotePath(collectionName);
             if (action == REMOTEQUERY) {
               action = ADMIN_OR_REMOTEQUERY;
               coreUrl = coreUrl.replace("/solr/", "/solr/____v2/c/");

solr/core/src/java/org/apache/solr/cli/ExportTool.java

@@ -660,8 +660,8 @@ class CoreHandler {
       }
 
       boolean exportDocsFromCore() throws IOException, SolrServerException {
-
-        try (SolrClient client = CLIUtils.getSolrClient(baseurl, credentials)) {
+        // reference the replica's node URL, not the baseUrl in scope, which could be anywhere
+        try (SolrClient client = CLIUtils.getSolrClient(replica.getBaseUrl(), credentials)) {
           expectedDocs = getDocCount(replica.getCoreName(), client, query);
           QueryRequest request;
           ModifiableSolrParams params = new ModifiableSolrParams();

solr/core/src/java/org/apache/solr/cli/RunExampleTool.java

@@ -647,10 +647,15 @@ protected Map<String, Object> startSolr(
     if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath))
       solrHome = solrHome.substring(cwdPath.length() + 1);
 
+    final var syspropArg =
+        ("techproducts".equals(cli.getOptionValue(EXAMPLE_OPTION)))
+            ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true"
+            : "";
+
     String startCmd =
         String.format(
             Locale.ROOT,
-            "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s",
+            "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s %s",
             callScript,
             cloudModeArg,
             port,
@@ -661,7 +666,8 @@ protected Map<String, Object> startSolr(
             forceArg,
             verboseArg,
             extraArgs,
-            jvmOptsArg);
+            jvmOptsArg,
+            syspropArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
 
     echo("\nStarting up Solr on port " + port + " using command:");

solr/core/src/java/org/apache/solr/core/ConfigSetService.java

@@ -272,7 +272,7 @@ public final ConfigSet loadConfigSet(CoreDescriptor dcore) {
       NamedList<?> properties = loadConfigSetProperties(dcore, coreLoader);
       boolean trusted = isConfigSetTrusted(coreLoader);
 
-      SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted);
+      SolrConfig solrConfig = createSolrConfig(dcore, coreLoader);
       return new ConfigSet(
           configSetName(dcore),
           solrConfig,
@@ -314,13 +314,12 @@ public ConfigSetService(SolrResourceLoader loader, boolean shareSchema) {
    *
    * @param cd the core's CoreDescriptor
    * @param loader the core's resource loader
-   * @param isTrusted is the configset trusted?
    * @return a SolrConfig object
    */
-  protected SolrConfig createSolrConfig(
-      CoreDescriptor cd, SolrResourceLoader loader, boolean isTrusted) throws IOException {
+  protected SolrConfig createSolrConfig(CoreDescriptor cd, SolrResourceLoader loader)
+      throws IOException {
     return SolrConfig.readFromResourceLoader(
-        loader, cd.getConfigName(), isTrusted, cd.getSubstitutableProperties());
+        loader, cd.getConfigName(), cd.getSubstitutableProperties());
   }
 
   /**

solr/core/src/java/org/apache/solr/core/SolrConfig.java

@@ -17,7 +17,6 @@
 package org.apache.solr.core;
 
 import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.params.CommonParams.PATH;
 import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
 import static org.apache.solr.core.SolrConfig.PluginOpts.LAZY;
 import static org.apache.solr.core.SolrConfig.PluginOpts.MULTI_OK;
@@ -31,7 +30,6 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.lang.invoke.MethodHandles;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -96,7 +94,7 @@
 
 /**
  * Provides a static reference to a Config object modeling the main configuration data for a Solr
- * instance -- typically found in "solrconfig.xml".
+ * core -- typically found in "solrconfig.xml".
  */
 public class SolrConfig implements MapSerializable {
 
@@ -143,16 +141,13 @@ public enum PluginOpts {
    * @param name the configuration name used by the loader if the stream is null
    */
   public SolrConfig(Path instanceDir, String name) throws IOException {
-    this(new SolrResourceLoader(instanceDir), name, true, null);
+    this(new SolrResourceLoader(instanceDir), name, null);
   }
 
   public static SolrConfig readFromResourceLoader(
-      SolrResourceLoader loader,
-      String name,
-      boolean isConfigsetTrusted,
-      Properties substitutableProperties) {
+      SolrResourceLoader loader, String name, Properties substitutableProperties) {
     try {
-      return new SolrConfig(loader, name, isConfigsetTrusted, substitutableProperties);
+      return new SolrConfig(loader, name, substitutableProperties);
     } catch (Exception e) {
       String resource;
       if (loader instanceof ZkSolrResourceLoader) {
@@ -196,15 +191,9 @@ public InputStream apply(String s) {
    *
    * @param loader the resource loader
    * @param name the configuration name
-   * @param isConfigsetTrusted false if configset was uploaded using unsecured configset upload API,
-   *     true otherwise
    * @param substitutableProperties optional properties to substitute into the XML
    */
-  private SolrConfig(
-      SolrResourceLoader loader,
-      String name,
-      boolean isConfigsetTrusted,
-      Properties substitutableProperties) {
+  private SolrConfig(SolrResourceLoader loader, String name, Properties substitutableProperties) {
     this.resourceLoader = loader;
     this.resourceName = name;
     this.substituteProperties = substitutableProperties;
@@ -237,7 +226,7 @@ private SolrConfig(
       rootDataHashCode = this.root.txt().hashCode();
 
       getRequestParams();
-      initLibs(loader, isConfigsetTrusted);
+      initLibs(loader);
       String val =
           root.child(
                   IndexSchema.LUCENE_MATCH_VERSION_PARAM,
@@ -934,11 +923,10 @@ public PluginInfo getPluginInfo(String type) {
         SolrException.ErrorCode.SERVER_ERROR, "Multiple plugins configured for type: " + type);
   }
 
-  private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
+  private void initLibs(SolrResourceLoader loader) {
     // TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass.
     // For Zk subclass, it's needed for the time being as well.  We could remove that one if we
-    // remove two things in SolrCloud: (1) instancePath/lib  and (2) solrconfig lib directives with
-    // relative paths. Can wait till 9.0.
+    // remove "instancePath/lib" in SolrCloud. Can wait till 9.0.
     Path instancePath = loader.getInstancePath();
     List<URL> urls = new ArrayList<>();
 
@@ -950,48 +938,15 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
         log.warn("Couldn't add files from {} to classpath: {}", libPath, e);
       }
     }
-
-    List<ConfigNode> nodes = root.getAll("lib");
-    if (nodes != null && nodes.size() > 0) {
-      if (!isConfigsetTrusted) {
-        throw new SolrException(
-            ErrorCode.UNAUTHORIZED,
-            "The configset for this collection was uploaded without any authentication in place,"
-                + " and use of <lib> is not available for collections with untrusted configsets. To use this component, re-upload the configset"
-                + " after enabling authentication and authorization.");
-      }
-
-      for (int i = 0; i < nodes.size(); i++) {
-        ConfigNode node = nodes.get(i);
-        String baseDir = node.attr("dir");
-        String path = node.attr(PATH);
-        if (null != baseDir) {
-          // :TODO: add support for a simpler 'glob' mutually exclusive of regex
-          Path dir = instancePath.resolve(baseDir);
-          String regex = node.attr("regex");
-          try {
-            if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir));
-            else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
-          } catch (IOException e) {
-            log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e);
-          }
-        } else if (null != path) {
-          final Path dir = instancePath.resolve(path);
-          try {
-            urls.add(dir.toUri().toURL());
-          } catch (MalformedURLException e) {
-            log.warn("Couldn't add file {} to classpath: {}", dir, e);
-          }
-        } else {
-          throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
-        }
-      }
-    }
-
     if (!urls.isEmpty()) {
       loader.addToClassLoader(urls);
       loader.reloadLuceneSPI();
     }
+
+    List<ConfigNode> nodes = root.getAll("lib");
+    if (nodes != null && nodes.size() > 0) {
+      log.warn("<lib/> entries no longer supported in solrconfig.xml; ignoring...");
+    }
   }
 
   public int getMultipartUploadLimitKB() {

solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java

@@ -76,7 +76,6 @@
 import org.apache.solr.client.api.model.SolrJerseyResponse;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.CoreAdminParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
@@ -347,14 +346,6 @@ private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp, SolrQue
       return;
     }
 
-    if (solrParams.getParams(CommonParams.WT) == null) {
-      reportErrorOnResponse(
-          rsp,
-          "Missing wt parameter",
-          new SolrException(SolrException.ErrorCode.BAD_REQUEST, "wt not specified in request"));
-      return;
-    }
-
     coreReplicationAPI.fetchFile(
         fileName,
         dirType,

solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java

@@ -254,7 +254,7 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp)
       try {
         InMemoryResourceLoader loader =
             new InMemoryResourceLoader(coreContainer, mutableId, SOLR_CONFIG_XML, data);
-        SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, requestIsTrusted, null);
+        SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, null);
       } catch (Exception exc) {
         updateFileError = exc;
       }

solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java

@@ -677,9 +677,7 @@ ManagedIndexSchema deleteNestedDocsFieldsIfNeeded(ManagedIndexSchema schema, boo
 
   SolrConfig loadSolrConfig(String configSet) {
     ZkSolrResourceLoader zkLoader = zkLoaderForConfigSet(configSet);
-    boolean trusted = isConfigSetTrusted(configSet);
-
-    return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, trusted, null);
+    return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, null);
   }
 
   ManagedIndexSchema loadLatestSchema(String configSet) {

solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java

@@ -27,6 +27,7 @@
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.core.NodeConfig;
@@ -85,6 +86,9 @@ public String toString() {
   /** Allow list of hosts. Elements in the list will be host:port (no protocol or context). */
   private final Set<String> hostAllowList;
 
+  private volatile Set<String> liveHostUrlsCache;
+  private volatile Set<String> liveNodesCache;
+
   /**
    * @param urlAllowList List of allowed URLs. URLs must be well-formed, missing protocol is
    *     tolerated. An empty list means there is no explicit allow-list of URLs, in this case no URL
@@ -136,11 +140,10 @@ public void checkAllowList(List<String> urls) throws MalformedURLException {
    */
   public void checkAllowList(List<String> urls, ClusterState clusterState)
       throws MalformedURLException {
-    Set<String> clusterHostAllowList =
-        clusterState == null ? Collections.emptySet() : clusterState.getHostAllowList();
+    Set<String> liveHostUrls = getLiveHostUrls(clusterState);
     for (String url : urls) {
       String hostPort = parseHostPort(url);
-      if (clusterHostAllowList.stream().noneMatch(hostPort::equalsIgnoreCase)
+      if (liveHostUrls.stream().noneMatch(hostPort::equalsIgnoreCase)
           && hostAllowList.stream().noneMatch(hostPort::equalsIgnoreCase)) {
         throw new SolrException(
             SolrException.ErrorCode.FORBIDDEN,
@@ -154,6 +157,33 @@ public void checkAllowList(List<String> urls, ClusterState clusterState)
     }
   }
 
+  /**
+   * Gets the set of live hosts urls (host:port) built from the set of live nodes. The set is cached
+   * to be reused until the live nodes change.
+   */
+  private Set<String> getLiveHostUrls(ClusterState clusterState) {
+    if (clusterState == null) {
+      return Set.of();
+    }
+    if (liveHostUrlsCache == null || clusterState.getLiveNodes() != liveNodesCache) {
+      synchronized (this) {
+        Set<String> liveNodes = clusterState.getLiveNodes();
+        if (liveHostUrlsCache == null || liveNodes != liveNodesCache) {
+          liveHostUrlsCache = buildLiveHostUrls(liveNodes);
+          liveNodesCache = liveNodes;
+        }
+      }
+    }
+    return liveHostUrlsCache;
+  }
+
+  @VisibleForTesting
+  Set<String> buildLiveHostUrls(Set<String> liveNodes) {
+    return liveNodes.stream()
+        .map((liveNode) -> liveNode.substring(0, liveNode.indexOf('_')))
+        .collect(Collectors.toSet());
+  }
+
   /** Whether this checker has been created with a non-empty allow-list of URLs. */
   public boolean hasExplicitAllowList() {
     return !hostAllowList.isEmpty();