diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java new file mode 100644 index 00000000000..d95a59ecf65 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import java.io.InputStream; +import java.util.List; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; + +@Path("/cluster") +public interface ClusterFileStoreApis { + // TODO Better understand the purpose of the 'sig' parameter and improve docs here. + @PUT + @Operation( + summary = "Upload a file to the filestore.", + tags = {"file-store"}) + @Path("/files{filePath:.+}") + UploadToFileStoreResponse uploadFile( + @Parameter(description = "File store path") @PathParam("filePath") String filePath, + @Parameter(description = "Signature(s) for the file being uploaded") @QueryParam("sig") + List sig, + @Parameter(description = "File content to be stored in the filestore") @RequestBody + InputStream requestBody); + + @DELETE + @Operation( + summary = "Delete a file or directory from the filestore.", + tags = {"file-store"}) + @Path("/files{path:.+}") + SolrJerseyResponse deleteFile( + @Parameter(description = "Path to a file or directory within the filestore") + @PathParam("path") + String path); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java new file mode 100644 index 00000000000..4492dce7daa --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class UploadToFileStoreResponse extends SolrJerseyResponse { + + @JsonProperty public String file; + @JsonProperty public String message; +} diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 452a6a086fb..7524135d92b 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -106,6 +106,9 @@ import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.BackupRepositoryFactory; +import org.apache.solr.filestore.ClusterFileStore; +import org.apache.solr.filestore.DistribFileStore; +import org.apache.solr.filestore.FileStore; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.handler.ClusterAPI; import org.apache.solr.handler.RequestHandlerBase; @@ -298,7 +301,9 @@ && getZkController().getOverseer() != null private volatile ClusterEventProducer clusterEventProducer; private DelegatingPlacementPluginFactory placementPluginFactory; + private DistribFileStore fileStore; private FileStoreAPI fileStoreAPI; + private ClusterFileStore clusterFileStoreAPI; private SolrPackageLoader packageLoader; private final Set allowPaths; @@ -727,8 +732,8 @@ public SolrPackageLoader getPackageLoader() { return packageLoader; } - public FileStoreAPI getFileStoreAPI() { - return fileStoreAPI; + public FileStore getFileStore() { + return fileStore; } public SolrCache getCache(String name) { @@ -860,9 +865,11 @@ private void loadInternal() { (PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH)); pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki"); + fileStore = new DistribFileStore(this); fileStoreAPI = new FileStoreAPI(this); registerV2ApiIfEnabled(fileStoreAPI.readAPI); registerV2ApiIfEnabled(fileStoreAPI.writeAPI); + registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); @@ -1154,6 +1161,15 @@ protected void configure() { .in(Singleton.class); } }) + .register( + new AbstractBinder() { + @Override + protected void configure() { + bindFactory(new InjectionFactories.SingletonFactory<>(fileStore)) + .to(DistribFileStore.class) + .in(Singleton.class); + } + }) .register( new AbstractBinder() { @Override diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java new file mode 100644 index 00000000000..a664e5699cb --- /dev/null +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.filestore; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.ClusterFileStoreApis; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.UploadToFileStoreResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.PermissionNameProvider; +import org.apache.solr.util.CryptoKeys; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ClusterFileStore extends JerseyResource implements ClusterFileStoreApis { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String FILESTORE_DIRECTORY = "filestore"; + public static final String TRUSTED_DIR = "_trusted_"; + public static final String KEYS_DIR = "/_trusted_/keys"; + static final String TMP_ZK_NODE = "/fileStoreWriteInProgress"; + + private final CoreContainer coreContainer; + private final SolrQueryRequest req; + private final SolrQueryResponse rsp; + private final FileStore fileStore; + + @Inject + public ClusterFileStore( + CoreContainer coreContainer, + DistribFileStore fileStore, + SolrQueryRequest req, + SolrQueryResponse rsp) { + this.coreContainer = coreContainer; + this.req = req; + this.rsp = rsp; + this.fileStore = fileStore; + } + + @Override + @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) + public UploadToFileStoreResponse uploadFile( + String filePath, List sig, InputStream requestBody) { + final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); + if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + throw new RuntimeException(PackageAPI.ERR_MSG); + } + try { + coreContainer + .getZkController() + .getZkClient() + .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); + + if (requestBody == null) + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload"); + if (filePath == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path"); + } + validateName(filePath, true); + try { + byte[] buf = requestBody.readAllBytes(); + List signatures = readSignatures(sig, buf); + FileStoreAPI.MetaData meta = _createJsonMetaData(buf, signatures); + FileStore.FileType type = fileStore.getType(filePath, true); + if (type == FileStore.FileType.FILE) { + // a file already exist at the same path + fileStore.get( + filePath, + fileEntry -> { + if (meta.equals(fileEntry.meta)) { + // the file content is same too. this is an idempotent put + // do not throw an error + response.file = filePath; + response.message = "File with same metadata exists "; + } + }, + true); + // 'message' only set in the "already exists w/ same content" case, so we're done! + if (response.message != null) { + return response; + } + } else if (type != FileStore.FileType.NOFILE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + filePath); + } + + fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, filePath)); + response.file = filePath; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } catch (InterruptedException e) { + log.error("Unexpected error", e); + } catch (KeeperException.NodeExistsException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later"); + } catch (KeeperException e) { + log.error("Unexpected error", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e.getMessage()); + } finally { + try { + coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); + } catch (Exception e) { + log.error("Unexpected error ", e); + } + } + + return response; + } + + @Override + @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) + public SolrJerseyResponse deleteFile(String filePath) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + throw new RuntimeException(PackageAPI.ERR_MSG); + } + + try { + coreContainer + .getZkController() + .getZkClient() + .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); + validateName(filePath, true); + if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); + } + FileStore.FileType type = fileStore.getType(filePath, true); + if (type == FileStore.FileType.NOFILE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath); + } + fileStore.delete(filePath); + return response; + } catch (SolrException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error", e); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } finally { + try { + coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); + } catch (Exception e) { + log.error("Unexpected error ", e); + } + } + } + + private List readSignatures(List signatures, byte[] buf) + throws SolrException, IOException { + if (signatures == null || signatures.isEmpty()) return null; + fileStore.refresh(KEYS_DIR); + validate(signatures, buf); + return signatures; + } + + private void validate(List sigs, byte[] buf) throws SolrException, IOException { + Map keys = fileStore.getKeys(); + if (keys == null || keys.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys"); + } + CryptoKeys cryptoKeys = null; + try { + cryptoKeys = new CryptoKeys(keys); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store"); + } + for (String sig : sigs) { + if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Signature does not match any public key : " + + sig + + " len: " + + buf.length + + " content sha512: " + + DigestUtils.sha512Hex(buf)); + } + } + } + + /** + * Creates a JSON string with the metadata. + * + * @lucene.internal + */ + public static FileStoreAPI.MetaData _createJsonMetaData(byte[] buf, List signatures) + throws IOException { + String sha512 = DigestUtils.sha512Hex(buf); + Map vals = new HashMap<>(); + vals.put(FileStoreAPI.MetaData.SHA512, sha512); + if (signatures != null) { + vals.put("sig", signatures); + } + return new FileStoreAPI.MetaData(vals); + } + + static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$> parts = StrUtils.splitSmart(path, '/', true); + for (String part : parts) { + if (part.charAt(0) == '.') { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period"); + } + for (int i = 0; i < part.length(); i++) { + for (int j = 0; j < INVALIDCHARS.length(); j++) { + if (part.charAt(i) == INVALIDCHARS.charAt(j)) + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part); + } + } + } + if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java index 4c62c1ff20e..7eb73354e6e 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java @@ -216,8 +216,9 @@ private boolean fetchFileFromNodeAndPersist(String fromNode) { } boolean fetchFromAnyNode() { - ArrayList l = coreContainer.getFileStoreAPI().shuffledNodes(); - for (String liveNode : l) { + ArrayList nodesToAttemptFetchFrom = + FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); + for (String liveNode : nodesToAttemptFetchFrom) { try { String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(liveNode); @@ -244,12 +245,6 @@ boolean fetchFromAnyNode() { return false; } - String getSimpleName() { - int idx = path.lastIndexOf('/'); - if (idx == -1) return path; - return path.substring(idx + 1); - } - public Path realPath() { return getRealpath(path); } @@ -306,6 +301,13 @@ public void writeMap(EntryWriter ew) throws IOException { ew.put("timestamp", getTimeStamp()); if (metaData != null) metaData.writeMap(ew); } + + @Override + public String getSimpleName() { + int idx = path.lastIndexOf('/'); + if (idx == -1) return path; + return path.substring(idx + 1); + } }; } @@ -353,7 +355,7 @@ private void distribute(FileInfo info) { } tmpFiles.put(info.path, info); - List nodes = coreContainer.getFileStoreAPI().shuffledNodes(); + List nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); int i = 0; int FETCHFROM_SRC = 50; String myNodeName = coreContainer.getZkController().getNodeName(); @@ -496,7 +498,7 @@ public List list(String path, Predicate predicate) { @Override public void delete(String path) { deleteLocal(path); - List nodes = coreContainer.getFileStoreAPI().shuffledNodes(); + List nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer); HttpClient client = coreContainer.getUpdateShardHandler().getDefaultHttpClient(); for (String node : nodes) { String baseUrl = @@ -583,7 +585,7 @@ public static boolean isMetaDataFile(String file) { } public static synchronized Path getFileStoreDirPath(Path solrHome) { - var path = solrHome.resolve(FileStoreAPI.FILESTORE_DIRECTORY); + var path = solrHome.resolve(ClusterFileStore.FILESTORE_DIRECTORY); if (!Files.exists(path)) { try { Files.createDirectories(path); @@ -632,7 +634,7 @@ public Map getKeys() throws IOException { // reads local keys file private static Map _getKeys(Path solrHome) throws IOException { Map result = new HashMap<>(); - Path keysDir = _getRealPath(FileStoreAPI.KEYS_DIR, solrHome); + Path keysDir = _getRealPath(ClusterFileStore.KEYS_DIR, solrHome); File[] keyFiles = keysDir.toFile().listFiles(); if (keyFiles == null) return result; diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStore.java b/solr/core/src/java/org/apache/solr/filestore/FileStore.java index fdff79e1fa5..65673757438 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStore.java @@ -111,6 +111,8 @@ enum FileType { interface FileDetails extends MapWriter { + String getSimpleName(); + MetaData getMetaData(); Date getTimeStamp(); diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java index 52dee0f5e9a..d54acc464f5 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java @@ -23,18 +23,11 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; -import java.util.function.Consumer; import java.util.function.Supplier; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.api.EndPoint; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.MapWriter; @@ -42,19 +35,14 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.util.CryptoKeys; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,101 +59,11 @@ public class FileStoreAPI { public FileStoreAPI(CoreContainer coreContainer) { this.coreContainer = coreContainer; - fileStore = new DistribFileStore(coreContainer); - } - - public FileStore getFileStore() { - return fileStore; - } - - /** get a list of nodes randomly shuffled * @lucene.internal */ - public ArrayList shuffledNodes() { - Set liveNodes = - coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - l.remove(coreContainer.getZkController().getNodeName()); - Collections.shuffle(l, BlobRepository.RANDOM); - return l; - } - - public void validateFiles(List files, boolean validateSignatures, Consumer errs) { - for (String path : files) { - try { - FileStore.FileType type = fileStore.getType(path, true); - if (type != FileStore.FileType.FILE) { - errs.accept("No such file: " + path); - continue; - } - - fileStore.get( - path, - entry -> { - if (entry.getMetaData().signatures == null - || entry.getMetaData().signatures.isEmpty()) { - errs.accept(path + " has no signature"); - return; - } - if (validateSignatures) { - try { - fileStore.refresh(KEYS_DIR); - validate(entry.meta.signatures, entry, false); - } catch (Exception e) { - log.error("Error validating package artifact", e); - errs.accept(e.getMessage()); - } - } - }, - false); - } catch (Exception e) { - log.error("Error reading file ", e); - errs.accept("Error reading file " + path + " " + e.getMessage()); - } - } + fileStore = coreContainer.getFileStore(); } public class FSWrite { - static final String TMP_ZK_NODE = "/fileStoreWriteInProgress"; - - @EndPoint( - path = "/cluster/files/*", - method = SolrRequest.METHOD.DELETE, - permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM) - public void delete(SolrQueryRequest req, SolrQueryResponse rsp) { - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); - } - - try { - coreContainer - .getZkController() - .getZkClient() - .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); - String path = req.getPathTemplateValues().get("*"); - validateName(path, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(path)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); - } - FileStore.FileType type = fileStore.getType(path, true); - if (type == FileStore.FileType.NOFILE) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + path); - } - fileStore.delete(path); - } catch (SolrException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error", e); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } finally { - try { - coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); - } catch (Exception e) { - log.error("Unexpected error ", e); - } - } - } - @EndPoint( path = "/node/files/*", method = SolrRequest.METHOD.DELETE, @@ -175,129 +73,6 @@ public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) { validateName(path, true); fileStore.deleteLocal(path); } - - @EndPoint( - path = "/cluster/files/*", - method = SolrRequest.METHOD.PUT, - permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM) - public void upload(SolrQueryRequest req, SolrQueryResponse rsp) { - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); - } - try { - coreContainer - .getZkController() - .getZkClient() - .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); - - Iterable streams = req.getContentStreams(); - if (streams == null) - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload"); - String path = req.getPathTemplateValues().get("*"); - if (path == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path"); - } - validateName(path, true); - ContentStream stream = streams.iterator().next(); - try { - byte[] buf = stream.getStream().readAllBytes(); - List signatures = readSignatures(req, buf); - MetaData meta = _createJsonMetaData(buf, signatures); - FileStore.FileType type = fileStore.getType(path, true); - boolean[] returnAfter = new boolean[] {false}; - if (type == FileStore.FileType.FILE) { - // a file already exist at the same path - fileStore.get( - path, - fileEntry -> { - if (meta.equals(fileEntry.meta)) { - // the file content is same too. this is an idempotent put - // do not throw an error - returnAfter[0] = true; - rsp.add(CommonParams.FILE, path); - rsp.add("message", "File with same metadata exists "); - } - }, - true); - } - if (returnAfter[0]) return; - if (type != FileStore.FileType.NOFILE) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + path); - } - fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, path)); - rsp.add(CommonParams.FILE, path); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - } catch (InterruptedException e) { - log.error("Unexpected error", e); - } catch (KeeperException.NodeExistsException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later"); - } catch (KeeperException e) { - log.error("Unexpected error", e); - } finally { - try { - coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true); - } catch (Exception e) { - log.error("Unexpected error ", e); - } - } - } - - private List readSignatures(SolrQueryRequest req, byte[] buf) - throws SolrException, IOException { - String[] signatures = req.getParams().getParams("sig"); - if (signatures == null || signatures.length == 0) return null; - List sigs = Arrays.asList(signatures); - fileStore.refresh(KEYS_DIR); - validate(sigs, buf); - return sigs; - } - - private void validate(List sigs, byte[] buf) throws SolrException, IOException { - Map keys = fileStore.getKeys(); - if (keys == null || keys.isEmpty()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys"); - } - CryptoKeys cryptoKeys = null; - try { - cryptoKeys = new CryptoKeys(keys); - } catch (Exception e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store"); - } - for (String sig : sigs) { - if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Signature does not match any public key : " - + sig - + " len: " - + buf.length - + " content sha512: " - + DigestUtils.sha512Hex(buf)); - } - } - } - } - - /** - * Creates a JSON string with the metadata. - * - * @lucene.internal - */ - public static MetaData _createJsonMetaData(byte[] buf, List signatures) - throws IOException { - String sha512 = DigestUtils.sha512Hex(buf); - Map vals = new HashMap<>(); - vals.put(MetaData.SHA512, sha512); - if (signatures != null) { - vals.put("sig", signatures); - } - return new MetaData(vals); } public class FSRead { diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java new file mode 100644 index 00000000000..98c33a72e54 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.filestore; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Supplier; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.BlobRepository; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.CryptoKeys; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Common utilities used by filestore-related code. */ +public class FileStoreUtils { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** Returns a shuffled list of all live nodes except the current host */ + public static ArrayList fetchAndShuffleRemoteLiveNodes(CoreContainer coreContainer) { + Set liveNodes = + coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + l.remove(coreContainer.getZkController().getNodeName()); + Collections.shuffle(l, BlobRepository.RANDOM); + return l; + } + + public static void validateFiles( + FileStore fileStore, List files, boolean validateSignatures, Consumer errs) { + for (String path : files) { + try { + FileStore.FileType type = fileStore.getType(path, true); + if (type != FileStore.FileType.FILE) { + errs.accept("No such file: " + path); + continue; + } + + fileStore.get( + path, + entry -> { + if (entry.getMetaData().signatures == null + || entry.getMetaData().signatures.isEmpty()) { + errs.accept(path + " has no signature"); + return; + } + if (validateSignatures) { + try { + fileStore.refresh(ClusterFileStore.KEYS_DIR); + validate(fileStore, entry.meta.signatures, entry, false); + } catch (Exception e) { + log.error("Error validating package artifact", e); + errs.accept(e.getMessage()); + } + } + }, + false); + } catch (Exception e) { + log.error("Error reading file ", e); + errs.accept("Error reading file " + path + " " + e.getMessage()); + } + } + } + + /** + * Validate a file for signature + * + * @param sigs the signatures, at least one should succeed + * @param entry The file details + * @param isFirstAttempt If there is a failure + */ + public static void validate( + FileStore fileStore, List sigs, FileStore.FileEntry entry, boolean isFirstAttempt) + throws SolrException, IOException { + if (!isFirstAttempt) { + // we are retrying because last validation failed. + // get all keys again and try again + fileStore.refresh(ClusterFileStore.KEYS_DIR); + } + + Map keys = fileStore.getKeys(); + if (keys == null || keys.isEmpty()) { + if (isFirstAttempt) { + validate(fileStore, sigs, entry, false); + return; + } + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Filestore does not have any public keys"); + } + CryptoKeys cryptoKeys = null; + try { + cryptoKeys = new CryptoKeys(keys); + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in ZooKeeper"); + } + for (String sig : sigs) { + Supplier errMsg = + () -> + "Signature does not match any public key : " + + sig + + "sha256 " + + entry.getMetaData().sha512; + if (entry.getBuffer() != null) { + if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { + if (isFirstAttempt) { + validate(fileStore, sigs, entry, false); + return; + } + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); + } + } else { + InputStream inputStream = entry.getInputStream(); + if (cryptoKeys.verify(sig, inputStream) == null) { + if (isFirstAttempt) { + validate(fileStore, sigs, entry, false); + return; + } + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); + } + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index d6e765b0d4a..15a5adc6246 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -49,6 +49,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.core.BlobRepository; +import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.DistribFileStore; import org.apache.solr.filestore.FileStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Manifest; @@ -302,7 +303,7 @@ public static String getCollectionParamsPath(String collection) { } public static void uploadKey(byte[] bytes, String path, Path home) throws IOException { - FileStoreAPI.MetaData meta = FileStoreAPI._createJsonMetaData(bytes, null); + FileStoreAPI.MetaData meta = ClusterFileStore._createJsonMetaData(bytes, null); DistribFileStore._persistToFile( home, path, ByteBuffer.wrap(bytes), ByteBuffer.wrap(Utils.toJSON(meta))); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 109468f854c..72d5f6c5db7 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -51,7 +51,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.BlobRepository; -import org.apache.solr.filestore.FileStoreAPI; +import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.PackageAPI; @@ -146,7 +146,7 @@ public void addKey(byte[] key, String destinationKeyFilename) throws Exception { String solrHome = (String) systemInfo.get("solr_home"); // put the public key into package store's trusted key store and request a sync. - String path = FileStoreAPI.KEYS_DIR + "/" + destinationKeyFilename; + String path = ClusterFileStore.KEYS_DIR + "/" + destinationKeyFilename; PackageUtils.uploadKey(key, path, Paths.get(solrHome)); PackageUtils.getJsonStringFromNonCollectionApi( solrClient, "/api/node/files" + path, new ModifiableSolrParams().add("sync", "true")); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 7e2c5a6f73e..24339feefe8 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -45,7 +45,7 @@ import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.filestore.FileStoreAPI; +import org.apache.solr.filestore.FileStoreUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.SolrJacksonAnnotationInspector; @@ -73,10 +73,6 @@ public class PackageAPI { public final Read readAPI = new Read(); public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { - if (coreContainer.getFileStoreAPI() == null) { - throw new IllegalStateException("Must successfully load FileStoreAPI first"); - } - this.coreContainer = coreContainer; this.packageLoader = loader; pkgs = new Packages(); @@ -254,10 +250,10 @@ public void refresh(PayloadObj payload) { } // first refresh my own packageLoader.notifyListeners(p); - for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { Utils.executeGET( coreContainer.getUpdateShardHandler().getDefaultHttpClient(), - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s) + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode) + "/cluster/package?wt=javabin&omitHeader=true&refreshPackage=" + p, Utils.JAVABINCONSUMER); @@ -272,8 +268,8 @@ public void add(PayloadObj payload) { payload.addError("No files specified"); return; } - FileStoreAPI fileStoreAPI = coreContainer.getFileStoreAPI(); - fileStoreAPI.validateFiles(add.files, true, s -> payload.addError(s)); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); if (payload.hasError()) return; Packages[] finalState = new Packages[1]; try { @@ -422,10 +418,10 @@ private void syncToVersion(int expectedVersion) { } void notifyAllNodesToSync(int expected) { - for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { Utils.executeGET( coreContainer.getUpdateShardHandler().getDefaultHttpClient(), - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s) + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode) + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected, Utils.JAVABINCONSUMER); diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index 5fe62e1f46b..0d5294f1ae6 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -42,6 +42,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.filestore.FileStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -272,12 +273,13 @@ public void writeMap(EntryWriter ew) throws IOException { List paths = new ArrayList<>(); List errs = new ArrayList<>(); - coreContainer.getFileStoreAPI().validateFiles(version.files, true, s -> errs.add(s)); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), version.files, true, s -> errs.add(s)); if (!errs.isEmpty()) { throw new RuntimeException("Cannot load package: " + errs); } for (String file : version.files) { - paths.add(coreContainer.getFileStoreAPI().getFileStore().getRealpath(file)); + paths.add(coreContainer.getFileStore().getRealpath(file)); } loader = diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 62668789d22..f7d9144d812 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -83,9 +83,8 @@ public void testFileStoreManagement() throws Exception { .addConfig("conf", configset("cloud-minimal")) .configure(); try { - byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); try { postFile( diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index faa07395241..a403febf549 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -53,7 +53,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.filestore.FileStoreAPI; +import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; import org.apache.solr.pkg.PackageAPI; @@ -295,7 +295,7 @@ public void testApiFromPackage() throws Exception { int version = phaser.getPhase(); byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); TestPackages.postFileAndWait( cluster, "runtimecode/containerplugin.v.1.jar.bin", diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index aae36019a65..04423132451 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -68,7 +68,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.filestore.FileStoreAPI; +import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.request.SolrQueryRequest; @@ -121,7 +121,7 @@ public void testCoreReloadingPlugin() throws Exception { String FILE1 = "/mypkg/runtimelibs.jar"; String COLLECTION_NAME = "testCoreReloadingPluginColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", @@ -190,7 +190,7 @@ public void testPluginLoading() throws Exception { String EXPR1 = "/mypkg/expressible.jar"; String COLLECTION_NAME = "testPluginLoadingColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/runtimelibs.jar.bin", @@ -609,7 +609,7 @@ public void testAPI() throws Exception { expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature"); // now we upload the keys byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); // and upload the same file with a different name, but it has proper signature postFileAndWait( cluster, @@ -755,7 +755,7 @@ public void testSchemaPlugins() throws Exception { String FILE1 = "/schemapkg/schema-plugins.jar"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); postFileAndWait( cluster, "runtimecode/schema-plugins.jar.bin",