Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-17582 Stream CLUSTERSTATUS API response #2916

Merged
merged 12 commits into from
Jan 4, 2025
Prev Previous commit
Next Next commit
Use MapWriter
mlbiscoc committed Dec 18, 2024
commit 6ed4a95b6399f9f867c1c7cc1861c6fa84dfb1b5
137 changes: 86 additions & 51 deletions solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.admin;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -28,6 +29,7 @@
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
@@ -206,64 +208,97 @@ private void fetchClusterStatusForCollOrAlias(
}
}

Iterator<NamedList<Object>> collectionPropsIt =
new Iterator<>() {
final Iterator<DocCollection> it = collectionStream.iterator();

@Override
public boolean hasNext() {
return it.hasNext();
MapWriter mw =
ew -> {
SolrCollectionPropsStreamer scps =
new SolrCollectionPropsStreamer(
collectionStream.iterator(), collectionVsAliases, routeKey, liveNodes, shard);
while (scps.hasNext()) {
Map<String, Object> output = (scps.next().asMap());
output.forEach(
(key, value) -> {
try {
ew.put(key, value);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
};
clusterStatus.add("collections", mw);
}

@Override
public NamedList<Object> next() {
NamedList<Object> collectionProps = new SimpleOrderedMap<>();
DocCollection clusterStateCollection = it.next();
Map<String, Object> collectionStatus;
String name = clusterStateCollection.getName();

Set<String> requestedShards = new HashSet<>();
if (routeKey != null) {
DocRouter router = clusterStateCollection.getRouter();
Collection<Slice> slices =
router.getSearchSlices(routeKey, null, clusterStateCollection);
for (Slice slice : slices) {
requestedShards.add(slice.getName());
}
}
if (shard != null) {
String[] paramShards = shard.split(",");
requestedShards.addAll(Arrays.asList(paramShards));
}
public class SolrCollectionPropsStreamer implements Iterator<NamedList<Object>> {

final Iterator<DocCollection> it;
Map<String, List<String>> collectionVsAliases;
String routeKey;
List<String> liveNodes;
String shard;

public SolrCollectionPropsStreamer(
Iterator<DocCollection> it,
Map<String, List<String>> collectionVsAliases,
String routeKey,
List<String> liveNodes,
String shard) {
this.it = it;
this.collectionVsAliases = collectionVsAliases;
this.routeKey = routeKey;
this.liveNodes = liveNodes;
this.shard = shard;
}

@Override
public boolean hasNext() {
return it.hasNext();
}

byte[] bytes = Utils.toJSON(clusterStateCollection);
@SuppressWarnings("unchecked")
Map<String, Object> docCollection = (Map<String, Object>) Utils.fromJSON(bytes);
collectionStatus = getCollectionStatus(docCollection, name, requestedShards);
@Override
public NamedList<Object> next() {
NamedList<Object> collectionProps = new SimpleOrderedMap<>();
DocCollection clusterStateCollection = it.next();
Map<String, Object> collectionStatus;
String name = clusterStateCollection.getName();

Set<String> requestedShards = new HashSet<>();
if (routeKey != null) {
DocRouter router = clusterStateCollection.getRouter();
Collection<Slice> slices = router.getSearchSlices(routeKey, null, clusterStateCollection);
for (Slice slice : slices) {
requestedShards.add(slice.getName());
}
}
if (shard != null) {
String[] paramShards = shard.split(",");
requestedShards.addAll(Arrays.asList(paramShards));
}

collectionStatus.put("znodeVersion", clusterStateCollection.getZNodeVersion());
collectionStatus.put(
"creationTimeMillis", clusterStateCollection.getCreationTime().toEpochMilli());
byte[] bytes = Utils.toJSON(clusterStateCollection);
@SuppressWarnings("unchecked")
Map<String, Object> docCollection = (Map<String, Object>) Utils.fromJSON(bytes);
collectionStatus = getCollectionStatus(docCollection, name, requestedShards);

if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
collectionStatus.put("aliases", collectionVsAliases.get(name));
}
String configName = clusterStateCollection.getConfigName();
collectionStatus.put("configName", configName);
if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
PerReplicaStates prs = clusterStateCollection.getPerReplicaStates();
collectionStatus.put("PRS", prs);
}
collectionProps.add(name, collectionStatus);
collectionStatus.put("znodeVersion", clusterStateCollection.getZNodeVersion());
collectionStatus.put(
"creationTimeMillis", clusterStateCollection.getCreationTime().toEpochMilli());

// now we need to walk the collectionProps tree to cross-check replica state with live
// nodes
crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);
return collectionProps;
}
};
if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
collectionStatus.put("aliases", collectionVsAliases.get(name));
}
String configName = clusterStateCollection.getConfigName();
collectionStatus.put("configName", configName);
if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
PerReplicaStates prs = clusterStateCollection.getPerReplicaStates();
collectionStatus.put("PRS", prs);
}
collectionProps.add(name, collectionStatus);

clusterStatus.add("collections", collectionPropsIt);
// now we need to walk the collectionProps tree to cross-check replica state with live
// nodes
crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);
return collectionProps;
}
}

private void addAliasMap(Aliases aliases, NamedList<Object> clusterStatus) {