diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c3aab06..26cfad08d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.19.3 - 2021-11-11 + +See [fix-counts.js](scripts/updates/fix-count.js) for a script that can be run before this update (as well as the +update for 1.19.0) to pre populate the migration. This will speed up the update and will not impact the running +instance. + +### Fixed +- If a space has a lot of datasets, rendering the space page is very slow. Files in a space is now cached. +- Set permissions for folders to be 777 this fixes [clowder-helm#5](https://github.com/clowder-framework/clowder-helm/issues/5) + ## 1.19.2 - 2021-10-20 ### Fixed diff --git a/Dockerfile b/Dockerfile index 6a7c9feb5..c663dec5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ COPY docker/custom.conf docker/play.plugins /home/clowder/custom/ # Containers should NOT run as root as a good practice # numeric id to be compatible with openshift, will run as random userid:0 RUN mkdir -p /home/clowder/data && \ - chmod g+w /home/clowder/logs /home/clowder/data /home/clowder/custom + chmod 777 /home/clowder/logs /home/clowder/data /home/clowder/custom USER 10001 # command to run when starting docker diff --git a/app/api/Files.scala b/app/api/Files.scala index 7f141c4ec..6aba93af3 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -1678,6 +1678,7 @@ class Files @Inject()( val ds_spaces = ds.spaces for (ds_s <- ds_spaces) { spaces.decrementSpaceBytes(ds_s, file.length) + spaces.decrementFileCounter(ds_s, 1) } } diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala index d82ba2c5e..36894f105 100644 --- a/app/api/Spaces.scala +++ b/app/api/Spaces.scala @@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService, val userId = request.user.get.id val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, - datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) + datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) spaces.insert(c) match { case Some(id) => { appConfig.incrementCount('spaces, 1) diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala index 0f527f3db..0cbedeec3 100644 --- a/app/controllers/SecuredController.scala +++ b/app/controllers/SecuredController.scala @@ -153,7 +153,7 @@ trait SecuredController extends Controller { val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService]) spaces.get(id) match { case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user))) - case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user))) + case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user))) } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index c42b38fb7..885750209 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -167,8 +167,6 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS var creatorActual: User = null val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size)) val datasetsInSpace = datasets.listSpace(size, id.toString(), user) - val spaceBytes : Long = s.spaceBytes - val spaceFiles : Integer = getFilesPerSpace(id, user) val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user) val usersInSpace = spaces.getUsersInSpace(id, None) var curationObjectsInSpace: List[CurationObject] = List() @@ -217,7 +215,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => List.empty } sinkService.logSpaceViewEvent(s, user) - Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles)) + Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections)) } case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } @@ -414,7 +412,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS val newSpace = ProjectSpace(name = formData.name, description = formData.description, created = new Date, creator = userId, homePage = formData.homePage, logoURL = formData.logoURL, bannerURL = formData.bannerURL, - collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, + collectionCount = 0, datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled, status = formData.access, affiliatedSpaces = formData.affSpace) @@ -640,15 +638,4 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } } - - private def getFilesPerSpace(spaceId: UUID, user: Option[User]) : Integer = { - var spaceFiles: Integer = 0 - val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), user) - for (ds <- allDatasetsInSpace) { - val files_in_ds = ds.files.length - spaceFiles += files_in_ds - } - spaceFiles - } - } diff --git a/app/models/Space.scala b/app/models/Space.scala index 6f2d545e1..8b4edc823 100644 --- a/app/models/Space.scala +++ b/app/models/Space.scala @@ -22,6 +22,7 @@ case class ProjectSpace ( bannerURL: Option[URL], collectionCount: Integer, datasetCount: Integer, + fileCount: Integer, userCount: Integer, spaceBytes: Long, metadata: List[Metadata], diff --git a/app/services/SpaceService.scala b/app/services/SpaceService.scala index 3a9f1e60b..d7cf8e5f7 100644 --- a/app/services/SpaceService.scala +++ b/app/services/SpaceService.scala @@ -94,6 +94,10 @@ trait SpaceService { def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int) + def incrementFileCounter(space: UUID, increment: Long) + + def decrementFileCounter(space: UUID, decrement: Long) + def incrementSpaceBytes(space: UUID, increment: Long) def decrementSpaceBytes(space: UUID, decrement: Long) diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index dac16f0c1..2f456f7c1 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -1399,6 +1399,15 @@ class MongoDBDatasetService @Inject() ( } for (f <- dataset.files) { val notTheDataset = for (currDataset <- findByFileIdDirectlyContain(f) if !dataset.id.toString.equals(currDataset.id.toString)) yield currDataset + files.get(f) match { + case Some(file) => { + for(space <- dataset.spaces) { + spaces.decrementFileCounter(space, 1) + spaces.decrementSpaceBytes(space, file.length) + } + } + case None => Logger.error(s"Error file with with id ${f} no longer exists") + } if (notTheDataset.size == 0) files.removeFile(f, host, apiKey, user) } diff --git a/app/services/mongodb/MongoDBSpaceService.scala b/app/services/mongodb/MongoDBSpaceService.scala index 6f164f8d0..c7dd53292 100644 --- a/app/services/mongodb/MongoDBSpaceService.scala +++ b/app/services/mongodb/MongoDBSpaceService.scala @@ -375,7 +375,15 @@ class MongoDBSpaceService @Inject() ( } def decrementCollectionCounter(collection: UUID, space: UUID, decrement: Int): Unit = { - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) + } + + def incrementFileCounter(space: UUID, increment: Long ): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> increment), upsert=false, multi=false, WriteConcern.Safe) + } + + def decrementFileCounter(space: UUID, decrement: Long): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) } def incrementSpaceBytes(space: UUID, increment: Long ): Unit = { @@ -383,7 +391,7 @@ class MongoDBSpaceService @Inject() ( } def decrementSpaceBytes(space: UUID, decrement: Long): Unit = { - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) } def removeCollection(collection:UUID, space:UUID): Unit = { @@ -398,11 +406,15 @@ class MongoDBSpaceService @Inject() ( */ def addDataset(dataset: UUID, space: UUID): Unit = { log.debug(s"Space Service - Adding $dataset to $space") - val datasetBytes = datasets.getBytesForDataset(dataset) - datasets.addToSpace(dataset, space) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) - + datasets.get(dataset) match { + case Some(x) => { + val datasetBytes = datasets.getBytesForDataset(dataset) + datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> x.files.length), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) + } + } } /** @@ -414,9 +426,15 @@ class MongoDBSpaceService @Inject() ( def removeDataset(dataset:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $dataset from $space") datasets.removeFromSpace(dataset, space) - val datasetBytes = datasets.getBytesForDataset(dataset) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + datasets.get(dataset) match { + case Some(x) => { + val datasetBytes = datasets.getBytesForDataset(dataset) + datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * datasetBytes), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * x.files.length), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + } + } } /** diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala index b5ede2229..3a625ae6c 100644 --- a/app/services/mongodb/MongoSalatPlugin.scala +++ b/app/services/mongodb/MongoSalatPlugin.scala @@ -452,6 +452,7 @@ class MongoSalatPlugin(app: Application) extends Plugin { // Adds space bytes to space updateMongo(updateKey = "update-space-bytes", updateSpaceBytes) + updateMongo(updateKey = "update-space-files", updateSpaceFiles) } private def updateMongo(updateKey: String, block: () => Unit): Unit = { @@ -513,7 +514,8 @@ class MongoSalatPlugin(app: Application) extends Plugin { val spacename = java.net.InetAddress.getLocalHost.getHostName val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"), homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata], - collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0) + collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, fileCount = 0, + spaceBytes = 0) ProjectSpaceDAO.save(newspace) val spaceId = new ObjectId(newspace.id.stringify) @@ -1707,4 +1709,16 @@ class MongoSalatPlugin(app: Application) extends Plugin { collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes)) } } + + private def updateSpaceFiles(): Unit = { + collection("spaces.projects").find().toList.foreach{ space => + var fileCount: Integer = 0 + val spaceId = space.get("_id") + val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList + spaceDatasets.foreach{ spaceDataset => + fileCount += spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty).length + } + collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("fileCount" -> fileCount)) + } + } } diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala index 949fa6dab..1d66d2f93 100644 --- a/app/util/FileUtils.scala +++ b/app/util/FileUtils.scala @@ -595,6 +595,7 @@ object FileUtils { val datasetSpaces = dataset.get.spaces for (s <- datasetSpaces){ spaceService.incrementSpaceBytes(s, file.length) + spaceService.incrementFileCounter(s, 1) } } } diff --git a/app/views/spaces/space.scala.html b/app/views/spaces/space.scala.html index 28ca667b2..696b81112 100644 --- a/app/views/spaces/space.scala.html +++ b/app/views/spaces/space.scala.html @@ -1,4 +1,4 @@ -@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String], spaceBytes: Long, spaceFiles: Integer)(implicit user: Option[models.User]) +@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String])(implicit user: Option[models.User]) @import play.api.libs.json._ @import play.api.Play.current @@ -119,10 +119,10 @@

@Messages("collections.title")

@* right column, space statistics, actions *@
@if(user.isDefined) { - @spaces.statistics(space, spaceBytes, None, spaceFiles ) + @spaces.statistics(space, None ) @spaces.otherActions(space) } else { - @spaces.statistics(space, spaceBytes, None, spaceFiles) + @spaces.statistics(space, None) } @spaces.externalLinks(space.homePage, space, "row ds-section-sm") @if(play.Play.application().configuration().getBoolean("enablePublic")) { diff --git a/app/views/spaces/statistics.scala.html b/app/views/spaces/statistics.scala.html index 24a388ef9..1e595d6b2 100644 --- a/app/views/spaces/statistics.scala.html +++ b/app/views/spaces/statistics.scala.html @@ -1,4 +1,4 @@ -@(space: ProjectSpace, spaceBytes: Long, isPublic: Option[Boolean], spaceFiles: Integer)(implicit user: Option[models.User]) +@(space: ProjectSpace, isPublic: Option[Boolean])(implicit user: Option[models.User]) @import play.api.i18n.Messages @import _root_.util.Formatters._ @@ -12,23 +12,23 @@

Statistics

@Messages("collections.title") - @space.collectionCount + @humanReadableNumber(space.collectionCount) @Messages("datasets.title") - @space.datasetCount + @humanReadableNumber(space.datasetCount) Files: - @spaceFiles + @humanReadableNumber(space.fileCount) Bytes: - @humanReadableByteCount(spaceBytes) + @humanReadableByteCount(space.spaceBytes) Users: - @space.userCount + @humanReadableNumber(space.userCount) diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py index 6201ab22c..4c0b7b4c7 100644 --- a/doc/src/sphinx/conf.py +++ b/doc/src/sphinx/conf.py @@ -22,7 +22,7 @@ author = 'Luigi Marini' # The full version, including alpha/beta/rc tags -release = '1.19.2' +release = '1.19.3' # -- General configuration --------------------------------------------------- diff --git a/project/Build.scala b/project/Build.scala index e4f8b8c7e..b57bb9255 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -13,7 +13,7 @@ import NativePackagerKeys._ object ApplicationBuild extends Build { val appName = "clowder" - val version = "1.19.2" + val version = "1.19.3" val jvm = "1.7" def appVersion: String = { diff --git a/public/swagger.yml b/public/swagger.yml index d2edab111..41162c120 100644 --- a/public/swagger.yml +++ b/public/swagger.yml @@ -9,7 +9,7 @@ info: Clowder is a customizable and scalable data management system to support any data format and multiple research domains. It is under active development and deployed for a variety of research projects. - version: 1.19.2 + version: 1.19.3 termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos contact: name: Clowder diff --git a/scripts/updates/fix-counts.js b/scripts/updates/fix-counts.js index f041e6841..dce99d3b9 100644 --- a/scripts/updates/fix-counts.js +++ b/scripts/updates/fix-counts.js @@ -1,3 +1,8 @@ +db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-bytes"}}) +db.spaces.projects.update({"spaceBytes": {$exists : false}}, {$set: {'spaceBytes': NumberLong(0)}}) +db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-files"}}) +db.spaces.projects.update({"fileCount": {$exists : false}}, {$set: {'fileCount': 0}}) + db.app.configuration.update({"key" : "countof.users"}, {$set: { "value": NumberLong(db.social.users.count())}}) db.app.configuration.update({"key" : "countof.datasets"}, {$set: { "value": NumberLong(db.datasets.count())}}) db.app.configuration.update({"key" : "countof.files"}, {$set: { "value": NumberLong(db.uploads.count())}}) @@ -5,3 +10,19 @@ db.app.configuration.update({"key" : "countof.bytes"}, {$set: { "value": db.uplo db.app.configuration.update({"key" : "countof.collections"}, {$set: { "value": NumberLong(db.collections.count())}}) db.app.configuration.update({"key" : "countof.spaces"}, {$set: { "value": NumberLong(db.spaces.projects.count())}}) +db.spaces.projects.find().sort({"datasetCount": 1}).forEach(function(s) { + var spaceBytes = 0; + var filecount = 0; + db.datasets.find({"spaces": s._id}).forEach(function(d) { + d.files.forEach(function(fid) { + db.uploads.find({"_id": fid}).forEach(function(f) { + spaceBytes += f.length; + filecount += 1; + }); + }); + }); + s.spaceBytes = NumberLong(spaceBytes); + s.fileCount = filecount; + db.spaces.projects.save(s); + print(s["_id"] + " " + s.name + " = " + s.spaceBytes + " in " + s.fileCount + " files"); +}) diff --git a/test/integration/spaces/SpaceMongoDBSpec.scala b/test/integration/spaces/SpaceMongoDBSpec.scala index 2287945ed..f0edd43bf 100644 --- a/test/integration/spaces/SpaceMongoDBSpec.scala +++ b/test/integration/spaces/SpaceMongoDBSpec.scala @@ -24,6 +24,7 @@ class SpaceMongoDBSpec extends PlaySpec with OneServerPerSuite { bannerURL = None, collectionCount = 0, datasetCount = 0, + fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty