diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java deleted file mode 100644 index d95a59ecf65..00000000000 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.api.endpoint; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.QueryParam; -import java.io.InputStream; -import java.util.List; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.api.model.UploadToFileStoreResponse; - -@Path("/cluster") -public interface ClusterFileStoreApis { - // TODO Better understand the purpose of the 'sig' parameter and improve docs here. - @PUT - @Operation( - summary = "Upload a file to the filestore.", - tags = {"file-store"}) - @Path("/files{filePath:.+}") - UploadToFileStoreResponse uploadFile( - @Parameter(description = "File store path") @PathParam("filePath") String filePath, - @Parameter(description = "Signature(s) for the file being uploaded") @QueryParam("sig") - List sig, - @Parameter(description = "File content to be stored in the filestore") @RequestBody - InputStream requestBody); - - @DELETE - @Operation( - summary = "Delete a file or directory from the filestore.", - tags = {"file-store"}) - @Path("/files{path:.+}") - SolrJerseyResponse deleteFile( - @Parameter(description = "Path to a file or directory within the filestore") - @PathParam("path") - String path); -} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java deleted file mode 100644 index 4492dce7daa..00000000000 --- a/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.api.model; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class UploadToFileStoreResponse extends SolrJerseyResponse { - - @JsonProperty public String file; - @JsonProperty public String message; -} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 7524135d92b..452a6a086fb 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -106,9 +106,6 @@ import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.BackupRepositoryFactory; -import org.apache.solr.filestore.ClusterFileStore; -import org.apache.solr.filestore.DistribFileStore; -import org.apache.solr.filestore.FileStore; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.handler.ClusterAPI; import org.apache.solr.handler.RequestHandlerBase; @@ -301,9 +298,7 @@ && getZkController().getOverseer() != null private volatile ClusterEventProducer clusterEventProducer; private DelegatingPlacementPluginFactory placementPluginFactory; - private DistribFileStore fileStore; private FileStoreAPI fileStoreAPI; - private ClusterFileStore clusterFileStoreAPI; private SolrPackageLoader packageLoader; private final Set allowPaths; @@ -732,8 +727,8 @@ public SolrPackageLoader getPackageLoader() { return packageLoader; } - public FileStore getFileStore() { - return fileStore; + public FileStoreAPI getFileStoreAPI() { + return fileStoreAPI; } public SolrCache getCache(String name) { @@ -865,11 +860,9 @@ private void loadInternal() { (PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH)); pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki"); - fileStore = new DistribFileStore(this); fileStoreAPI = new FileStoreAPI(this); registerV2ApiIfEnabled(fileStoreAPI.readAPI); registerV2ApiIfEnabled(fileStoreAPI.writeAPI); - registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); @@ -1161,15 +1154,6 @@ protected void configure() { .in(Singleton.class); } }) - .register( - new AbstractBinder() { - @Override - protected void configure() { - bindFactory(new InjectionFactories.SingletonFactory<>(fileStore)) - .to(DistribFileStore.class) - .in(Singleton.class); - } - }) .register( new AbstractBinder() { @Override diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java deleted file mode 100644 index a664e5699cb..00000000000 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.filestore; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import jakarta.inject.Inject; -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.endpoint.ClusterFileStoreApis; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.client.api.model.UploadToFileStoreResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.security.PermissionNameProvider; -import org.apache.solr.util.CryptoKeys; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ClusterFileStore extends JerseyResource implements ClusterFileStoreApis { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String FILESTORE_DIRECTORY = "filestore"; - public static final String TRUSTED_DIR = "_trusted_"; - public static final String KEYS_DIR = "/_trusted_/keys"; - static final String TMP_ZK_NODE = "/fileStoreWriteInProgress"; - - private final CoreContainer coreContainer; - private final SolrQueryRequest req; - private final SolrQueryResponse rsp; - private final FileStore fileStore; - - @Inject - public ClusterFileStore( - CoreContainer coreContainer, - DistribFileStore fileStore, - SolrQueryRequest req, - SolrQueryResponse rsp) { - this.coreContainer = coreContainer; - this.req = req; - this.rsp = rsp; - this.fileStore = fileStore; - } - - @Override - @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) - public UploadToFileStoreResponse uploadFile( - String filePath, List sig, InputStream requestBody) { - final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); - } - try { - coreContainer - .getZkController() - .getZkClient() - .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); - - if (requestBody == null) - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload"); - if (filePath == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path"); - } - validateName(filePath, true); - try { - byte[] buf = requestBody.readAllBytes(); - List signatures = readSignatures(sig, buf); - FileStoreAPI.MetaData meta = _createJsonMetaData(buf, signatures); - FileStore.FileType type = fileStore.getType(filePath, true); - if (type == FileStore.FileType.FILE) { - // a file already exist at the same path - fileStore.get( - filePath, - fileEntry -> { - if (meta.equals(fileEntry.meta)) { - // the file content is same too. this is an idempotent put - // do not throw an error - response.file = filePath; - response.message = "File with same metadata exists "; - } - }, - true); - // 'message' only set in the "already exists w/ same content" case, so we're done! - if (response.message != null) { - return response; - } - } else if (type != FileStore.FileType.NOFILE) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + filePath); - } - - fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, filePath)); - response.file = filePath; - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - } catch (InterruptedException e) { - log.error("Unexpected error", e); - } catch (KeeperException.NodeExistsException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later"); - } catch (KeeperException e) { - log.error("Unexpected error", e); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e.getMessage()); - } finally { - try { - coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); - } catch (Exception e) { - log.error("Unexpected error ", e); - } - } - - return response; - } - - @Override - @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) - public SolrJerseyResponse deleteFile(String filePath) { - final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); - } - - try { - coreContainer - .getZkController() - .getZkClient() - .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); - validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); - } - FileStore.FileType type = fileStore.getType(filePath, true); - if (type == FileStore.FileType.NOFILE) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath); - } - fileStore.delete(filePath); - return response; - } catch (SolrException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error", e); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } finally { - try { - coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); - } catch (Exception e) { - log.error("Unexpected error ", e); - } - } - } - - private List readSignatures(List signatures, byte[] buf) - throws SolrException, IOException { - if (signatures == null || signatures.isEmpty()) return null; - fileStore.refresh(KEYS_DIR); - validate(signatures, buf); - return signatures; - } - - private void validate(List sigs, byte[] buf) throws SolrException, IOException { - Map keys = fileStore.getKeys(); - if (keys == null || keys.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys"); - } - CryptoKeys cryptoKeys = null; - try { - cryptoKeys = new CryptoKeys(keys); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store"); - } - for (String sig : sigs) { - if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Signature does not match any public key : " - + sig - + " len: " - + buf.length - + " content sha512: " - + DigestUtils.sha512Hex(buf)); - } - } - } - - /** - * Creates a JSON string with the metadata. - * - * @lucene.internal - */ - public static FileStoreAPI.MetaData _createJsonMetaData(byte[] buf, List signatures) - throws IOException { - String sha512 = DigestUtils.sha512Hex(buf); - Map vals = new HashMap<>(); - vals.put(FileStoreAPI.MetaData.SHA512, sha512); - if (signatures != null) { - vals.put("sig", signatures); - } - return new FileStoreAPI.MetaData(vals); - } - - static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$> parts = StrUtils.splitSmart(path, '/', true); - for (String part : parts) { - if (part.charAt(0) == '.') { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period"); - } - for (int i = 0; i < part.length(); i++) { - for (int j = 0; j < INVALIDCHARS.length(); j++) { - if (part.charAt(i) == INVALIDCHARS.charAt(j)) - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part); - } - } - } - if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory"); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java index 7eb73354e6e..4c62c1ff20e 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java @@ -216,9 +216,8 @@ private boolean fetchFileFromNodeAndPersist(String fromNode) { } boolean fetchFromAnyNode() { - ArrayList nodesToAttemptFetchFrom = - FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); - for (String liveNode : nodesToAttemptFetchFrom) { + ArrayList l = coreContainer.getFileStoreAPI().shuffledNodes(); + for (String liveNode : l) { try { String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(liveNode); @@ -245,6 +244,12 @@ boolean fetchFromAnyNode() { return false; } + String getSimpleName() { + int idx = path.lastIndexOf('/'); + if (idx == -1) return path; + return path.substring(idx + 1); + } + public Path realPath() { return getRealpath(path); } @@ -301,13 +306,6 @@ public void writeMap(EntryWriter ew) throws IOException { ew.put("timestamp", getTimeStamp()); if (metaData != null) metaData.writeMap(ew); } - - @Override - public String getSimpleName() { - int idx = path.lastIndexOf('/'); - if (idx == -1) return path; - return path.substring(idx + 1); - } }; } @@ -355,7 +353,7 @@ private void distribute(FileInfo info) { } tmpFiles.put(info.path, info); - List nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); + List nodes = coreContainer.getFileStoreAPI().shuffledNodes(); int i = 0; int FETCHFROM_SRC = 50; String myNodeName = coreContainer.getZkController().getNodeName(); @@ -498,7 +496,7 @@ public List list(String path, Predicate predicate) { @Override public void delete(String path) { deleteLocal(path); - List nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); + List nodes = coreContainer.getFileStoreAPI().shuffledNodes(); HttpClient client = coreContainer.getUpdateShardHandler().getDefaultHttpClient(); for (String node : nodes) { String baseUrl = @@ -585,7 +583,7 @@ public static boolean isMetaDataFile(String file) { } public static synchronized Path getFileStoreDirPath(Path solrHome) { - var path = solrHome.resolve(ClusterFileStore.FILESTORE_DIRECTORY); + var path = solrHome.resolve(FileStoreAPI.FILESTORE_DIRECTORY); if (!Files.exists(path)) { try { Files.createDirectories(path); @@ -634,7 +632,7 @@ public Map getKeys() throws IOException { // reads local keys file private static Map _getKeys(Path solrHome) throws IOException { Map result = new HashMap<>(); - Path keysDir = _getRealPath(ClusterFileStore.KEYS_DIR, solrHome); + Path keysDir = _getRealPath(FileStoreAPI.KEYS_DIR, solrHome); File[] keyFiles = keysDir.toFile().listFiles(); if (keyFiles == null) return result; diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStore.java b/solr/core/src/java/org/apache/solr/filestore/FileStore.java index 65673757438..fdff79e1fa5 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStore.java @@ -111,8 +111,6 @@ enum FileType { interface FileDetails extends MapWriter { - String getSimpleName(); - MetaData getMetaData(); Date getTimeStamp(); diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java index d54acc464f5..52dee0f5e9a 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java @@ -23,11 +23,18 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; import java.util.function.Supplier; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.api.EndPoint; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.MapWriter; @@ -35,14 +42,19 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.util.CryptoKeys; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,11 +71,101 @@ public class FileStoreAPI { public FileStoreAPI(CoreContainer coreContainer) { this.coreContainer = coreContainer; - fileStore = coreContainer.getFileStore(); + fileStore = new DistribFileStore(coreContainer); + } + + public FileStore getFileStore() { + return fileStore; + } + + /** get a list of nodes randomly shuffled * @lucene.internal */ + public ArrayList shuffledNodes() { + Set liveNodes = + coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + l.remove(coreContainer.getZkController().getNodeName()); + Collections.shuffle(l, BlobRepository.RANDOM); + return l; + } + + public void validateFiles(List files, boolean validateSignatures, Consumer errs) { + for (String path : files) { + try { + FileStore.FileType type = fileStore.getType(path, true); + if (type != FileStore.FileType.FILE) { + errs.accept("No such file: " + path); + continue; + } + + fileStore.get( + path, + entry -> { + if (entry.getMetaData().signatures == null + || entry.getMetaData().signatures.isEmpty()) { + errs.accept(path + " has no signature"); + return; + } + if (validateSignatures) { + try { + fileStore.refresh(KEYS_DIR); + validate(entry.meta.signatures, entry, false); + } catch (Exception e) { + log.error("Error validating package artifact", e); + errs.accept(e.getMessage()); + } + } + }, + false); + } catch (Exception e) { + log.error("Error reading file ", e); + errs.accept("Error reading file " + path + " " + e.getMessage()); + } + } } public class FSWrite { + static final String TMP_ZK_NODE = "/fileStoreWriteInProgress"; + + @EndPoint( + path = "/cluster/files/*", + method = SolrRequest.METHOD.DELETE, + permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM) + public void delete(SolrQueryRequest req, SolrQueryResponse rsp) { + if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + throw new RuntimeException(PackageAPI.ERR_MSG); + } + + try { + coreContainer + .getZkController() + .getZkClient() + .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); + String path = req.getPathTemplateValues().get("*"); + validateName(path, true); + if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(path)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); + } + FileStore.FileType type = fileStore.getType(path, true); + if (type == FileStore.FileType.NOFILE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + path); + } + fileStore.delete(path); + } catch (SolrException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } finally { + try { + coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); + } catch (Exception e) { + log.error("Unexpected error ", e); + } + } + } + @EndPoint( path = "/node/files/*", method = SolrRequest.METHOD.DELETE, @@ -73,6 +175,129 @@ public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) { validateName(path, true); fileStore.deleteLocal(path); } + + @EndPoint( + path = "/cluster/files/*", + method = SolrRequest.METHOD.PUT, + permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM) + public void upload(SolrQueryRequest req, SolrQueryResponse rsp) { + if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + throw new RuntimeException(PackageAPI.ERR_MSG); + } + try { + coreContainer + .getZkController() + .getZkClient() + .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); + + Iterable streams = req.getContentStreams(); + if (streams == null) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload"); + String path = req.getPathTemplateValues().get("*"); + if (path == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path"); + } + validateName(path, true); + ContentStream stream = streams.iterator().next(); + try { + byte[] buf = stream.getStream().readAllBytes(); + List signatures = readSignatures(req, buf); + MetaData meta = _createJsonMetaData(buf, signatures); + FileStore.FileType type = fileStore.getType(path, true); + boolean[] returnAfter = new boolean[] {false}; + if (type == FileStore.FileType.FILE) { + // a file already exist at the same path + fileStore.get( + path, + fileEntry -> { + if (meta.equals(fileEntry.meta)) { + // the file content is same too. this is an idempotent put + // do not throw an error + returnAfter[0] = true; + rsp.add(CommonParams.FILE, path); + rsp.add("message", "File with same metadata exists "); + } + }, + true); + } + if (returnAfter[0]) return; + if (type != FileStore.FileType.NOFILE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + path); + } + fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, path)); + rsp.add(CommonParams.FILE, path); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } catch (InterruptedException e) { + log.error("Unexpected error", e); + } catch (KeeperException.NodeExistsException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later"); + } catch (KeeperException e) { + log.error("Unexpected error", e); + } finally { + try { + coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); + } catch (Exception e) { + log.error("Unexpected error ", e); + } + } + } + + private List readSignatures(SolrQueryRequest req, byte[] buf) + throws SolrException, IOException { + String[] signatures = req.getParams().getParams("sig"); + if (signatures == null || signatures.length == 0) return null; + List sigs = Arrays.asList(signatures); + fileStore.refresh(KEYS_DIR); + validate(sigs, buf); + return sigs; + } + + private void validate(List sigs, byte[] buf) throws SolrException, IOException { + Map keys = fileStore.getKeys(); + if (keys == null || keys.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys"); + } + CryptoKeys cryptoKeys = null; + try { + cryptoKeys = new CryptoKeys(keys); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store"); + } + for (String sig : sigs) { + if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Signature does not match any public key : " + + sig + + " len: " + + buf.length + + " content sha512: " + + DigestUtils.sha512Hex(buf)); + } + } + } + } + + /** + * Creates a JSON string with the metadata. + * + * @lucene.internal + */ + public static MetaData _createJsonMetaData(byte[] buf, List signatures) + throws IOException { + String sha512 = DigestUtils.sha512Hex(buf); + Map vals = new HashMap<>(); + vals.put(MetaData.SHA512, sha512); + if (signatures != null) { + vals.put("sig", signatures); + } + return new MetaData(vals); } public class FSRead { diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java deleted file mode 100644 index 98c33a72e54..00000000000 --- a/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.filestore; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Supplier; -import org.apache.solr.common.SolrException; -import org.apache.solr.core.BlobRepository; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.util.CryptoKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Common utilities used by filestore-related code. */ -public class FileStoreUtils { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** Returns a shuffled list of all live nodes except the current host */ - public static ArrayList fetchAndShuffleRemoteLiveNodes(CoreContainer coreContainer) { - Set liveNodes = - coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - l.remove(coreContainer.getZkController().getNodeName()); - Collections.shuffle(l, BlobRepository.RANDOM); - return l; - } - - public static void validateFiles( - FileStore fileStore, List files, boolean validateSignatures, Consumer errs) { - for (String path : files) { - try { - FileStore.FileType type = fileStore.getType(path, true); - if (type != FileStore.FileType.FILE) { - errs.accept("No such file: " + path); - continue; - } - - fileStore.get( - path, - entry -> { - if (entry.getMetaData().signatures == null - || entry.getMetaData().signatures.isEmpty()) { - errs.accept(path + " has no signature"); - return; - } - if (validateSignatures) { - try { - fileStore.refresh(ClusterFileStore.KEYS_DIR); - validate(fileStore, entry.meta.signatures, entry, false); - } catch (Exception e) { - log.error("Error validating package artifact", e); - errs.accept(e.getMessage()); - } - } - }, - false); - } catch (Exception e) { - log.error("Error reading file ", e); - errs.accept("Error reading file " + path + " " + e.getMessage()); - } - } - } - - /** - * Validate a file for signature - * - * @param sigs the signatures, at least one should succeed - * @param entry The file details - * @param isFirstAttempt If there is a failure - */ - public static void validate( - FileStore fileStore, List sigs, FileStore.FileEntry entry, boolean isFirstAttempt) - throws SolrException, IOException { - if (!isFirstAttempt) { - // we are retrying because last validation failed. - // get all keys again and try again - fileStore.refresh(ClusterFileStore.KEYS_DIR); - } - - Map keys = fileStore.getKeys(); - if (keys == null || keys.isEmpty()) { - if (isFirstAttempt) { - validate(fileStore, sigs, entry, false); - return; - } - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Filestore does not have any public keys"); - } - CryptoKeys cryptoKeys = null; - try { - cryptoKeys = new CryptoKeys(keys); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in ZooKeeper"); - } - for (String sig : sigs) { - Supplier errMsg = - () -> - "Signature does not match any public key : " - + sig - + "sha256 " - + entry.getMetaData().sha512; - if (entry.getBuffer() != null) { - if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { - if (isFirstAttempt) { - validate(fileStore, sigs, entry, false); - return; - } - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); - } - } else { - InputStream inputStream = entry.getInputStream(); - if (cryptoKeys.verify(sig, inputStream) == null) { - if (isFirstAttempt) { - validate(fileStore, sigs, entry, false); - return; - } - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); - } - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index 15a5adc6246..d6e765b0d4a 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -49,7 +49,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.BlobRepository; -import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.DistribFileStore; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Manifest; @@ -303,7 +302,7 @@ public static String getCollectionParamsPath(String collection) { } public static void uploadKey(byte[] bytes, String path, Path home) throws IOException { - FileStoreAPI.MetaData meta = ClusterFileStore._createJsonMetaData(bytes, null); + FileStoreAPI.MetaData meta = FileStoreAPI._createJsonMetaData(bytes, null); DistribFileStore._persistToFile( home, path, ByteBuffer.wrap(bytes), ByteBuffer.wrap(Utils.toJSON(meta))); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 72d5f6c5db7..109468f854c 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -51,7 +51,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.BlobRepository; -import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.PackageAPI; @@ -146,7 +146,7 @@ public void addKey(byte[] key, String destinationKeyFilename) throws Exception { String solrHome = (String) systemInfo.get("solr_home"); // put the public key into package store's trusted key store and request a sync. - String path = ClusterFileStore.KEYS_DIR + "/" + destinationKeyFilename; + String path = FileStoreAPI.KEYS_DIR + "/" + destinationKeyFilename; PackageUtils.uploadKey(key, path, Paths.get(solrHome)); PackageUtils.getJsonStringFromNonCollectionApi( solrClient, "/api/node/files" + path, new ModifiableSolrParams().add("sync", "true")); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 24339feefe8..7e2c5a6f73e 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -45,7 +45,7 @@ import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.SolrJacksonAnnotationInspector; @@ -73,6 +73,10 @@ public class PackageAPI { public final Read readAPI = new Read(); public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { + if (coreContainer.getFileStoreAPI() == null) { + throw new IllegalStateException("Must successfully load FileStoreAPI first"); + } + this.coreContainer = coreContainer; this.packageLoader = loader; pkgs = new Packages(); @@ -250,10 +254,10 @@ public void refresh(PayloadObj payload) { } // first refresh my own packageLoader.notifyListeners(p); - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) { Utils.executeGET( coreContainer.getUpdateShardHandler().getDefaultHttpClient(), - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode) + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s) + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p, Utils.JAVABINCONSUMER); @@ -268,8 +272,8 @@ public void add(PayloadObj payload) { payload.addError("No files specified"); return; } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); + FileStoreAPI fileStoreAPI = coreContainer.getFileStoreAPI(); + fileStoreAPI.validateFiles(add.files, true, s -> payload.addError(s)); if (payload.hasError()) return; Packages[] finalState = new Packages[1]; try { @@ -418,10 +422,10 @@ private void syncToVersion(int expectedVersion) { } void notifyAllNodesToSync(int expected) { - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) { Utils.executeGET( coreContainer.getUpdateShardHandler().getDefaultHttpClient(), - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode) + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s) + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected, Utils.JAVABINCONSUMER); diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index 0d5294f1ae6..5fe62e1f46b 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -42,7 +42,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.filestore.FileStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,13 +272,12 @@ public void writeMap(EntryWriter ew) throws IOException { List paths = new ArrayList<>(); List errs = new ArrayList<>(); - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), version.files, true, s -> errs.add(s)); + coreContainer.getFileStoreAPI().validateFiles(version.files, true, s -> errs.add(s)); if (!errs.isEmpty()) { throw new RuntimeException("Cannot load package: " + errs); } for (String file : version.files) { - paths.add(coreContainer.getFileStore().getRealpath(file)); + paths.add(coreContainer.getFileStoreAPI().getFileStore().getRealpath(file)); } loader = diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index f7d9144d812..62668789d22 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -83,8 +83,9 @@ public void testFileStoreManagement() throws Exception { .addConfig("conf", configset("cloud-minimal")) .configure(); try { + byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); try { postFile( diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index a403febf549..faa07395241 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -53,7 +53,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; import org.apache.solr.pkg.PackageAPI; @@ -295,7 +295,7 @@ public void testApiFromPackage() throws Exception { int version = phaser.getPhase(); byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); TestPackages.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 04423132451..aae36019a65 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -68,7 +68,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; @@ -121,7 +121,7 @@ public void testCoreReloadingPlugin() throws Exception { String FILE1 = "/mypkg/runtimelibs.jar"; String COLLECTION_NAME = "testCoreReloadingPluginColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", @@ -190,7 +190,7 @@ public void testPluginLoading() throws Exception { String EXPR1 = "/mypkg/expressible.jar"; String COLLECTION_NAME = "testPluginLoadingColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", @@ -609,7 +609,7 @@ public void testAPI() throws Exception { expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); // now we upload the keys byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); // and upload the same file with a different name, but it has proper signature postFileAndWait( cluster, @@ -755,7 +755,7 @@ public void testSchemaPlugins() throws Exception { String FILE1 = "/schemapkg/schema-plugins.jar"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/schema-plugins.jar.bin",