diff --git a/build.gradle b/build.gradle index 7fe487ab..28335c49 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ allprojects { apply plugin: 'maven-publish' apply plugin: 'idea' group = 'org.janelia.jacs-model' - version = '2.41.2' + version = '2.42' } subprojects { diff --git a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/TmNeuronMetadataDao.java b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/TmNeuronMetadataDao.java index 2d15d8ef..70f22621 100644 --- a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/TmNeuronMetadataDao.java +++ b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/TmNeuronMetadataDao.java @@ -20,12 +20,9 @@ public interface TmNeuronMetadataDao extends DomainObjectDao { List getTmNeuronMetadataByWorkspaceId(TmWorkspace workspace, String subjectKey, long offset, int length); Iterable streamWorkspaceNeurons(TmWorkspace workspace, String subjectKey, long offset, int length); List getTmNeuronMetadataByNeuronIds(TmWorkspace workspace, List neuronList); - List> getTmNeuronsMetadataWithPointStreamsByWorkspaceId( - TmWorkspace workspace, String subjectKey, long offset, int length); boolean removeTmNeuron(Long neuronId, boolean isLarge, TmWorkspace workspace, String subjectKey); void updateNeuronStyles(BulkNeuronStyleUpdate bulkNeuronStyleUpdate, TmWorkspace workspace, String subjectKey); void removeEmptyNeuronsInWorkspace(TmWorkspace workspace, String subjectKey); - void bulkMigrateNeuronsInWorkspace(TmWorkspace workspace, Collection neurons, String subjectKey); void updateNeuronTagsForNeurons(TmWorkspace workspace, List neuronIds, List tags, boolean tagState, String subjectKey); Long getNeuronCountsForWorkspace(TmWorkspace workspace, String subjectKey); TmNeuronMetadata saveNeuronMetadata(TmWorkspace workspace, TmNeuronMetadata neuron, String subjectKey); diff --git a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/mongo/TmNeuronMetadataMongoDao.java b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/mongo/TmNeuronMetadataMongoDao.java index d88c0e2b..12f61847 100644 --- a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/mongo/TmNeuronMetadataMongoDao.java +++ b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/mongo/TmNeuronMetadataMongoDao.java @@ -22,12 +22,9 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.bson.conversions.Bson; import org.janelia.model.access.domain.DomainDAO; import org.janelia.model.access.domain.dao.*; -import org.janelia.model.domain.DomainUtils; import org.janelia.model.domain.Reference; import org.janelia.model.domain.tiledMicroscope.*; import org.janelia.model.access.domain.TimebasedIdentifierGenerator; @@ -357,70 +354,6 @@ public void removeEmptyNeuronsInWorkspace(TmWorkspace workspace, String subjectK )); } - @Override - public void bulkMigrateNeuronsInWorkspace(TmWorkspace workspace, Collection neurons, String subjectKey) { - String collectionName = MongoDaoHelper.findOrCreateCappedCollection (this, mongoDatabase, - "tmNeuron", 20000000000L, TmNeuronMetadata.class); - try { - workspace.setNeuronCollection(collectionName); - workspace = domainDao.save(subjectKey, workspace); - } catch (Exception e) { - LOG.error("ERROR SAVING WORKSPACE {} - Issue with Neuron Collection Key", workspace.getId(),e); - throw new RuntimeException("ERROR SAVING WORKSPACE" + workspace.getId() + " - Issue with Neuron Collection Key"); - } - - MongoCollection copyLoc = getNeuronCollection(collectionName); - if (neurons==null || neurons.isEmpty()) - return; - List neuronList = new ArrayList<>(neurons); - try { - int prevCount = 0; - for (int i=0; i(i+1000)) { - copyLoc.insertMany(neuronList.subList(i, i + 1000)); - LOG.info("Inserted {} neurons", i+1000); - } - else { - copyLoc.insertMany(neuronList.subList(i, neuronList.size())); - LOG.info("Inserted {} neurons", neuronList.size()); - } - } - LOG.info("Finished workspace {}",workspace.getId()); - } catch (org.bson.BsonMaximumSizeExceededException e) { - // one or more of the documents is too big - // save them to gridfs and then try again - int count = 0; - List largeNeurons = new ArrayList<>(); - List truncatedList = new ArrayList<>(); - boolean hitFirstLarge = false; - for (TmNeuronMetadata neuron: neuronList) { - if (hitFirstLarge) - truncatedList.add(neuron); - // remove records that were saved before batch failed - MongoDaoHelper.deleteMatchingRecords(copyLoc, - Filters.and(MongoDaoHelper.createFilterById(neuron.getId()), - permissionsHelper.createWritePermissionFilterForSubjectKey(subjectKey))); - boolean isLarge = checkLargeNeuron(neuron); - TmNeuronData pointData = neuron.getNeuronData(); - if (isLarge) { - hitFirstLarge = true; - largeNeurons.add(neuron.getId()); - neuron.setNeuronData(null); - saveLargeNeuronPointData(neuron.getId(), pointData); - neuron.setLargeNeuron(isLarge); - } - } - - try { - LOG.info("Large neurons in this batch are {}",largeNeurons); - if (!truncatedList.isEmpty()) - copyLoc.insertMany(truncatedList); - } catch (org.bson.BsonMaximumSizeExceededException ee) { - LOG.error("ERROR PROCESSING {} - Issue with Large Neurons", workspace.getId(),e); - } - } - } - @Override public void updateNeuronTagsForNeurons(TmWorkspace workspace, List neuronIds, List tags, boolean tagState, String subjectKey) { @@ -459,21 +392,4 @@ public Long getNeuronCountsForWorkspace(TmWorkspace workspace, String subjectKey private MongoCollection getNeuronCollection(String collectionName) { return mongoDatabase.getCollection(collectionName, TmNeuronMetadata.class); } - - @Override - public List> getTmNeuronsMetadataWithPointStreamsByWorkspaceId(TmWorkspace workspace, - String subjectKey, - long offset, int length) { - workspace.setNeuronCollection("tmNeuron"); - List workspaceNeurons = getTmNeuronMetadataByWorkspaceId(workspace,subjectKey, offset, length); - if (workspaceNeurons.isEmpty()) { - return Collections.emptyList(); - } - Map indexedWorkspaceNeurons = DomainUtils.getMapById(workspaceNeurons); - Map neuronsPointStreams = tmNeuronBufferDao.streamNeuronPointsByWorkspaceId(indexedWorkspaceNeurons.keySet(), workspace.getId()); - - return workspaceNeurons.stream() - .map(neuron -> ImmutablePair.of(neuron, neuronsPointStreams.get(neuron.getId()))) - .collect(Collectors.toList()); - } } diff --git a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/searchables/TmNeuronMetadataSearchableDao.java b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/searchables/TmNeuronMetadataSearchableDao.java index a5e92bb4..26f3bb0b 100644 --- a/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/searchables/TmNeuronMetadataSearchableDao.java +++ b/jacs-model-access/src/main/java/org/janelia/model/access/domain/dao/searchables/TmNeuronMetadataSearchableDao.java @@ -77,12 +77,6 @@ public void removeEmptyNeuronsInWorkspace(TmWorkspace workspace, String subjectK tmNeuronMetadataDao.removeEmptyNeuronsInWorkspace(workspace, subjectKey); } - @Override - public void bulkMigrateNeuronsInWorkspace(TmWorkspace workspace, Collection neurons, - String subjectKey) { - tmNeuronMetadataDao.bulkMigrateNeuronsInWorkspace(workspace, neurons, subjectKey); - } - @Override public void updateNeuronTagsForNeurons(TmWorkspace workspace, List neuronIds, List tags, boolean tagState, String subjectKey) { @@ -109,10 +103,4 @@ public List getOperations(Long workspaceId, Long neuronId, Date sta return tmNeuronMetadataDao.getOperations(workspaceId, neuronId, startDate, endDate); } - @Override - public List> getTmNeuronsMetadataWithPointStreamsByWorkspaceId(TmWorkspace workspace, - String subjectKey, long offset, int length) { - return tmNeuronMetadataDao.getTmNeuronsMetadataWithPointStreamsByWorkspaceId(workspace, subjectKey,offset,length); - } - } diff --git a/jacs-model-core/src/main/java/org/janelia/model/domain/tiledMicroscope/TmNeuronMetadata.java b/jacs-model-core/src/main/java/org/janelia/model/domain/tiledMicroscope/TmNeuronMetadata.java index 3e55b202..157ca231 100644 --- a/jacs-model-core/src/main/java/org/janelia/model/domain/tiledMicroscope/TmNeuronMetadata.java +++ b/jacs-model-core/src/main/java/org/janelia/model/domain/tiledMicroscope/TmNeuronMetadata.java @@ -192,6 +192,7 @@ public void initNeuronData() { for (Long geoId : neuronData.getGeoAnnotationMap().keySet()) { TmGeoAnnotation geoAnnotation = neuronData.getGeoAnnotationMap().get(geoId); geoAnnotation.setNeuronId(getId()); + geoAnnotation.setModificationDate(geoAnnotation.getCreationDate()); } }