SOLR-17302: Convert /cluster filestore APIs to JAX-RS (#2470)

This commit moves some of Solr's "filestore" APIs (those located under
the `/api/cluster/files` path) to JAX-RS.

solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.endpoint;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.QueryParam;
+import java.io.InputStream;
+import java.util.List;
+import org.apache.solr.client.api.model.SolrJerseyResponse;
+import org.apache.solr.client.api.model.UploadToFileStoreResponse;
+
+@Path("/cluster")
+public interface ClusterFileStoreApis {
+  // TODO Better understand the purpose of the 'sig' parameter and improve docs here.
+  @PUT
+  @Operation(
+      summary = "Upload a file to the filestore.",
+      tags = {"file-store"})
+  @Path("/files{filePath:.+}")
+  UploadToFileStoreResponse uploadFile(
+      @Parameter(description = "File store path") @PathParam("filePath") String filePath,
+      @Parameter(description = "Signature(s) for the file being uploaded") @QueryParam("sig")
+          List<String> sig,
+      @Parameter(description = "File content to be stored in the filestore") @RequestBody
+          InputStream requestBody);
+
+  @DELETE
+  @Operation(
+      summary = "Delete a file or directory from the filestore.",
+      tags = {"file-store"})
+  @Path("/files{path:.+}")
+  SolrJerseyResponse deleteFile(
+      @Parameter(description = "Path to a file or directory within the filestore")
+          @PathParam("path")
+          String path);
+}

solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class UploadToFileStoreResponse extends SolrJerseyResponse {
+
+  @JsonProperty public String file;
+  @JsonProperty public String message;
+}

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -106,6 +106,9 @@
 import org.apache.solr.core.DirectoryFactory.DirContext;
 import org.apache.solr.core.backup.repository.BackupRepository;
 import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
+import org.apache.solr.filestore.ClusterFileStore;
+import org.apache.solr.filestore.DistribFileStore;
+import org.apache.solr.filestore.FileStore;
 import org.apache.solr.filestore.FileStoreAPI;
 import org.apache.solr.handler.ClusterAPI;
 import org.apache.solr.handler.RequestHandlerBase;
@@ -298,7 +301,9 @@ && getZkController().getOverseer() != null
   private volatile ClusterEventProducer clusterEventProducer;
   private DelegatingPlacementPluginFactory placementPluginFactory;
 
+  private DistribFileStore fileStore;
   private FileStoreAPI fileStoreAPI;
+  private ClusterFileStore clusterFileStoreAPI;
   private SolrPackageLoader packageLoader;
 
   private final Set<Path> allowPaths;
@@ -727,8 +732,8 @@ public SolrPackageLoader getPackageLoader() {
     return packageLoader;
   }
 
-  public FileStoreAPI getFileStoreAPI() {
-    return fileStoreAPI;
+  public FileStore getFileStore() {
+    return fileStore;
   }
 
   public SolrCache<?, ?> getCache(String name) {
@@ -860,9 +865,11 @@ private void loadInternal() {
               (PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));
       pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki");
 
+      fileStore = new DistribFileStore(this);
       fileStoreAPI = new FileStoreAPI(this);
       registerV2ApiIfEnabled(fileStoreAPI.readAPI);
       registerV2ApiIfEnabled(fileStoreAPI.writeAPI);
+      registerV2ApiIfEnabled(ClusterFileStore.class);
 
       packageLoader = new SolrPackageLoader(this);
       registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);
@@ -1154,6 +1161,15 @@ protected void configure() {
                       .in(Singleton.class);
                 }
               })
+          .register(
+              new AbstractBinder() {
+                @Override
+                protected void configure() {
+                  bindFactory(new InjectionFactories.SingletonFactory<>(fileStore))
+                      .to(DistribFileStore.class)
+                      .in(Singleton.class);
+                }
+              })
           .register(
               new AbstractBinder() {
                 @Override

solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java

@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import jakarta.inject.Inject;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.JerseyResource;
+import org.apache.solr.client.api.endpoint.ClusterFileStoreApis;
+import org.apache.solr.client.api.model.SolrJerseyResponse;
+import org.apache.solr.client.api.model.UploadToFileStoreResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.jersey.PermissionName;
+import org.apache.solr.pkg.PackageAPI;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ClusterFileStore extends JerseyResource implements ClusterFileStoreApis {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  public static final String FILESTORE_DIRECTORY = "filestore";
+  public static final String TRUSTED_DIR = "_trusted_";
+  public static final String KEYS_DIR = "/_trusted_/keys";
+  static final String TMP_ZK_NODE = "/fileStoreWriteInProgress";
+
+  private final CoreContainer coreContainer;
+  private final SolrQueryRequest req;
+  private final SolrQueryResponse rsp;
+  private final FileStore fileStore;
+
+  @Inject
+  public ClusterFileStore(
+      CoreContainer coreContainer,
+      DistribFileStore fileStore,
+      SolrQueryRequest req,
+      SolrQueryResponse rsp) {
+    this.coreContainer = coreContainer;
+    this.req = req;
+    this.rsp = rsp;
+    this.fileStore = fileStore;
+  }
+
+  @Override
+  @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+  public UploadToFileStoreResponse uploadFile(
+      String filePath, List<String> sig, InputStream requestBody) {
+    final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class);
+    if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
+      throw new RuntimeException(PackageAPI.ERR_MSG);
+    }
+    try {
+      coreContainer
+          .getZkController()
+          .getZkClient()
+          .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);
+
+      if (requestBody == null)
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+      if (filePath == null) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path");
+      }
+      validateName(filePath, true);
+      try {
+        byte[] buf = requestBody.readAllBytes();
+        List<String> signatures = readSignatures(sig, buf);
+        FileStoreAPI.MetaData meta = _createJsonMetaData(buf, signatures);
+        FileStore.FileType type = fileStore.getType(filePath, true);
+        if (type == FileStore.FileType.FILE) {
+          // a file already exist at the same path
+          fileStore.get(
+              filePath,
+              fileEntry -> {
+                if (meta.equals(fileEntry.meta)) {
+                  // the file content is same too. this is an idempotent put
+                  // do not throw an error
+                  response.file = filePath;
+                  response.message = "File with same metadata exists ";
+                }
+              },
+              true);
+          // 'message' only set in the "already exists w/ same content" case, so we're done!
+          if (response.message != null) {
+            return response;
+          }
+        } else if (type != FileStore.FileType.NOFILE) {
+          throw new SolrException(
+              SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + filePath);
+        }
+
+        fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, filePath));
+        response.file = filePath;
+      } catch (IOException e) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+      }
+    } catch (InterruptedException e) {
+      log.error("Unexpected error", e);
+    } catch (KeeperException.NodeExistsException e) {
+      throw new SolrException(
+          SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later");
+    } catch (KeeperException e) {
+      log.error("Unexpected error", e);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e.getMessage());
+    } finally {
+      try {
+        coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);
+      } catch (Exception e) {
+        log.error("Unexpected error  ", e);
+      }
+    }
+
+    return response;
+  }
+
+  @Override
+  @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+  public SolrJerseyResponse deleteFile(String filePath) {
+    final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+    if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
+      throw new RuntimeException(PackageAPI.ERR_MSG);
+    }
+
+    try {
+      coreContainer
+          .getZkController()
+          .getZkClient()
+          .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);
+      validateName(filePath, true);
+      if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");
+      }
+      FileStore.FileType type = fileStore.getType(filePath, true);
+      if (type == FileStore.FileType.NOFILE) {
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);
+      }
+      fileStore.delete(filePath);
+      return response;
+    } catch (SolrException e) {
+      throw e;
+    } catch (Exception e) {
+      log.error("Unknown error", e);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+    } finally {
+      try {
+        coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);
+      } catch (Exception e) {
+        log.error("Unexpected error  ", e);
+      }
+    }
+  }
+
+  private List<String> readSignatures(List<String> signatures, byte[] buf)
+      throws SolrException, IOException {
+    if (signatures == null || signatures.isEmpty()) return null;
+    fileStore.refresh(KEYS_DIR);
+    validate(signatures, buf);
+    return signatures;
+  }
+
+  private void validate(List<String> sigs, byte[] buf) throws SolrException, IOException {
+    Map<String, byte[]> keys = fileStore.getKeys();
+    if (keys == null || keys.isEmpty()) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys");
+    }
+    CryptoKeys cryptoKeys = null;
+    try {
+      cryptoKeys = new CryptoKeys(keys);
+    } catch (Exception e) {
+      throw new SolrException(
+          SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store");
+    }
+    for (String sig : sigs) {
+      if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) {
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST,
+            "Signature does not match any public key : "
+                + sig
+                + " len: "
+                + buf.length
+                + " content sha512: "
+                + DigestUtils.sha512Hex(buf));
+      }
+    }
+  }
+
+  /**
+   * Creates a JSON string with the metadata.
+   *
+   * @lucene.internal
+   */
+  public static FileStoreAPI.MetaData _createJsonMetaData(byte[] buf, List<String> signatures)
+      throws IOException {
+    String sha512 = DigestUtils.sha512Hex(buf);
+    Map<String, Object> vals = new HashMap<>();
+    vals.put(FileStoreAPI.MetaData.SHA512, sha512);
+    if (signatures != null) {
+      vals.put("sig", signatures);
+    }
+    return new FileStoreAPI.MetaData(vals);
+  }
+
+  static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";
+
+  public static void validateName(String path, boolean failForTrusted) {
+    if (path == null) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");
+    }
+    List<String> parts = StrUtils.splitSmart(path, '/', true);
+    for (String part : parts) {
+      if (part.charAt(0) == '.') {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period");
+      }
+      for (int i = 0; i < part.length(); i++) {
+        for (int j = 0; j < INVALIDCHARS.length(); j++) {
+          if (part.charAt(i) == INVALIDCHARS.charAt(j))
+            throw new SolrException(
+                SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part);
+        }
+      }
+    }
+    if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory");
+    }
+  }
+}