Skip to content

Commit

Permalink
Merge pull request #2 from JaneliaSciComp/db_cleanup
Browse files Browse the repository at this point in the history
Admin DB Cleanup for TmWorkspaces
  • Loading branch information
porterbot authored Mar 4, 2025
2 parents a4606a3 + 126ed70 commit f7c2c9b
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 10 deletions.
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ allprojects {
apply plugin: 'idea'

group = 'org.janelia.jacs-model'
version = '3.3.0'
version = '3.3.4'
}

subprojects {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import org.janelia.model.domain.tiledMicroscope.BoundingBox3d;
import org.janelia.model.domain.tiledMicroscope.TmWorkspace;
import org.janelia.model.domain.tiledMicroscope.TmWorkspaceInfo;

import java.util.List;

Expand All @@ -16,5 +17,6 @@ public interface TmWorkspaceDao extends DomainObjectDao<TmWorkspace> {
TmWorkspace updateTmWorkspace(String subjectKey, TmWorkspace tmWorkspace);
void saveWorkspaceBoundingBoxes(TmWorkspace workspace, List<BoundingBox3d> boundingBoxes);
List<BoundingBox3d> getWorkspaceBoundingBoxes(Long workspaceId);

List<TmWorkspaceInfo> getLargestWorkspaces(String subjectKey, Long limit);
long deleteByIdAndSubjectKey(Long id, String subjectKey);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,15 @@
import org.janelia.model.domain.Reference;
import org.janelia.model.domain.tiledMicroscope.TmMappedNeuron;
import org.janelia.model.domain.tiledMicroscope.TmWorkspace;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.util.List;

public class TmMappedNeuronMongoDao extends AbstractDomainObjectMongoDao<TmMappedNeuron> implements TmMappedNeuronDao {
private static final Logger LOG = LoggerFactory.getLogger(TmMappedNeuronMongoDao.class);


@Inject
TmMappedNeuronMongoDao(MongoDatabase mongoDatabase,
Expand All @@ -36,7 +40,9 @@ public List<TmMappedNeuron> getNeuronsForWorkspace(TmWorkspace workspace) {

@Override
public long deleteNeuronsForWorkspace(TmWorkspace workspace, String subjectKey) {
// TODO: this should remove the deleted documents from the search index

LOG.info("Deleting neurons from workspace {} in mongo collection {}",
workspace.getName(), workspace.getNeuronCollection());
return MongoDaoHelper.deleteMatchingRecords(mongoCollection,
Filters.and(MongoDaoHelper.createFilterCriteria(
Filters.eq("workspaceRef", Reference.createFor(workspace))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,15 +310,12 @@ public List<TmOperation> getOperations(Long workspaceId, Long neuronId, Date sta

private TmNeuronMetadata createNeuronWithExistingId(TmNeuronMetadata entity,
String collectionName, String subjectKey) {
LOG.info("TRACE1");
MongoCollection<TmNeuronMetadata> mongoCollection = getNeuronCollection(collectionName);

Date now = new Date();
LOG.info("TRACE2");
for (TmGeoAnnotation anno : entity.getRootAnnotations()) {
anno.setParentId(entity.getId());
}
LOG.info("TRACE3");
entity.setOwnerKey(subjectKey);
entity.getReaders().add(subjectKey);
entity.getWriters().add(subjectKey);
Expand Down Expand Up @@ -448,10 +445,12 @@ public Long getNeuronCountsForWorkspace(TmWorkspace workspace, String subjectKey

@Override
public long deleteNeuronsForWorkspace(TmWorkspace workspace, String subjectKey) {
return MongoDaoHelper.deleteMatchingRecords(mongoCollection,
Filters.and(MongoDaoHelper.createFilterCriteria(
Filters.eq("workspaceRef", Reference.createFor(workspace))
), permissionsHelper.createWritePermissionFilterForSubjectKey(subjectKey)));
LOG.info("Deleting neurons from workspace {} in mongo collection {}",
workspace.getName(), workspace.getNeuronCollection());
return MongoDaoHelper.deleteMatchingRecords(
getNeuronCollection(workspace.getNeuronCollection()),
Filters.eq("workspaceRef", Reference.createFor(workspace)) // Simplified filter
);
}

private MongoCollection<TmNeuronMetadata> getNeuronCollection(String collectionName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,31 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.*;
import java.util.concurrent.ForkJoinPool;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import javax.inject.Inject;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Accumulators;
import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.Filters;

import com.mongodb.client.model.Projections;
import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
import org.janelia.model.access.domain.DomainDAO;
import org.janelia.model.access.domain.dao.TmMappedNeuronDao;
import org.janelia.model.access.domain.dao.TmNeuronMetadataDao;
import org.janelia.model.access.domain.dao.TmWorkspaceDao;
import org.janelia.model.domain.tiledMicroscope.TmWorkspaceInfo;
import org.janelia.model.domain.DomainConstants;
import org.janelia.model.domain.Reference;
import org.janelia.model.domain.tiledMicroscope.*;
Expand Down Expand Up @@ -88,6 +97,64 @@ public List<TmWorkspace> getAllTmWorkspaces(String subjectKey) {
TmWorkspace.class);
}

@Override
public List<TmWorkspaceInfo> getLargestWorkspaces(String subjectKey, Long limit) {
// Step 1: Get all accessible workspaces using existing method
List<TmWorkspace> workspaces = getAllTmWorkspaces(subjectKey);

List<TmWorkspaceInfo> workspaceInfoList = new ArrayList<>();

// Step 2: Use parallel processing for faster aggregation
ForkJoinPool customThreadPool = new ForkJoinPool(8); // Limit to 8 threads
workspaceInfoList = customThreadPool.submit(() ->
workspaces.parallelStream()
.map(workspace -> {
String neuronCollectionName = workspace.getNeuronCollection();
if (neuronCollectionName == null || neuronCollectionName.isEmpty()) {
return new TmWorkspaceInfo(workspace.getId(), workspace.getName(),
0L, workspace.getOwnerKey(), workspace.getCreationDate());
}

MongoCollection<Document> neuronCollection = mongoDatabase.getCollection(neuronCollectionName);

LOG.info("Analyzing {} workspace storing neurons in {}",
workspace.getName(), neuronCollectionName);

Long totalSize = 0L;
try {
String workspaceRef = "TmWorkspace#" + workspace.getId();
AggregateIterable<Document> aggregation = neuronCollection.aggregate(Arrays.asList(
Aggregates.match(Filters.eq("workspaceRef", workspaceRef)),
Aggregates.project(Projections.fields(
Projections.computed("size", new Document("$bsonSize", "$$ROOT"))
)),
Aggregates.group(null, Accumulators.sum("totalSize", "$size"))
));

for (Document result : aggregation) {
Number size = result.get("totalSize", Number.class);
totalSize = size != null ? size.longValue() : 0L;
}
} catch (Exception e) {
LOG.error("Error processing workspace: {}", workspace.getName(), e);
}

LOG.info("Finished aggregation analysis for {}", workspace.getName());
return new TmWorkspaceInfo(workspace.getId(), workspace.getName(),
totalSize, workspace.getOwnerKey(), workspace.getCreationDate());
})
.sorted((w1, w2) -> Long.compare(w2.getTotalSize(), w1.getTotalSize())) // Sort by total size in descending order
.limit(limit) // Limit the results
.collect(Collectors.toList())
).join();

customThreadPool.shutdown();
LOG.info("Workspace analysis complete. Results: {}", workspaceInfoList);

return workspaceInfoList;
}


@Override
public TmWorkspace createTmWorkspace(String subjectKey, TmWorkspace tmWorkspace) {
try {
Expand Down Expand Up @@ -246,6 +313,7 @@ public long delete(TmWorkspace workspace, String subjectKey) {
catch (Exception e) {
throw new IllegalStateException("Error deleting neuron metadata for workspace "+workspace, e);
}
gridFSMongoDao.deleteDataBlock(workspace.getId().toString());
n += super.deleteByIdAndSubjectKey(workspace.getId(), subjectKey);
return n;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.janelia.model.access.domain.search.DomainObjectIndexer;
import org.janelia.model.domain.tiledMicroscope.BoundingBox3d;
import org.janelia.model.domain.tiledMicroscope.TmWorkspace;
import org.janelia.model.domain.tiledMicroscope.TmWorkspaceInfo;

/**
* {@link TmWorkspace} DAO.
Expand All @@ -35,6 +36,11 @@ public List<TmWorkspace> getAllTmWorkspaces(String subjectKey) {
return tmWorkspaceDao.getAllTmWorkspaces(subjectKey);
}

@Override
public long deleteByIdAndSubjectKey(Long id, String subjectKey) {
return tmWorkspaceDao.deleteByIdAndSubjectKey(id, subjectKey);
}

@Override
public TmWorkspace createTmWorkspace(String subjectKey, TmWorkspace tmWorkspace) {
TmWorkspace persistedTmWorkspace = tmWorkspaceDao.createTmWorkspace(subjectKey, tmWorkspace);
Expand Down Expand Up @@ -65,4 +71,9 @@ public void saveWorkspaceBoundingBoxes(TmWorkspace workspace, List<BoundingBox3d
public List<BoundingBox3d> getWorkspaceBoundingBoxes(Long workspaceId) {
return tmWorkspaceDao.getWorkspaceBoundingBoxes(workspaceId);
}

@Override
public List<TmWorkspaceInfo> getLargestWorkspaces(String subjectKey, Long limit) {
return tmWorkspaceDao.getLargestWorkspaces(subjectKey, limit);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.janelia.model.domain.tiledMicroscope;

import java.util.Date;

public class TmWorkspaceInfo {
private Long workspaceId;
private String workspaceName;
private Long totalSize;
private String ownerKey;
private Date dateCreated;

public TmWorkspaceInfo() {
}

public TmWorkspaceInfo(Long workspaceId, String workspaceName, Long totalSize, String ownerKey, Date dateCreated) {
this.workspaceId = workspaceId;
this.workspaceName = workspaceName;
this.totalSize = totalSize;
this.ownerKey = ownerKey;
this.dateCreated = dateCreated;
}

public String getWorkspaceName() {
return workspaceName;
}

public void setWorkspaceName(String workspaceName) {
this.workspaceName = workspaceName;
}

public Date getDateCreated() {
return dateCreated;
}

public void setDateCreated(Date dateCreated) {
this.dateCreated = dateCreated;
}

public Long getWorkspaceId() {
return workspaceId;
}

public void setWorkspaceId(Long workspaceId) {
this.workspaceId = workspaceId;
}

public Long getTotalSize() {
return totalSize;
}

public void setTotalSize(Long totalSize) {
this.totalSize = totalSize;
}

public String getOwnerKey() {
return ownerKey;
}

public void setOwnerKey(String ownerKey) {
this.ownerKey = ownerKey;
}
}


0 comments on commit f7c2c9b

Please sign in to comment.