diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c3aab06..26cfad08d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.19.3 - 2021-11-11 + +See [fix-counts.js](scripts/updates/fix-count.js) for a script that can be run before this update (as well as the +update for 1.19.0) to pre populate the migration. This will speed up the update and will not impact the running +instance. + +### Fixed +- If a space has a lot of datasets, rendering the space page is very slow. Files in a space is now cached. +- Set permissions for folders to be 777 this fixes [clowder-helm#5](https://github.com/clowder-framework/clowder-helm/issues/5) + ## 1.19.2 - 2021-10-20 ### Fixed diff --git a/Dockerfile b/Dockerfile index 6a7c9feb5..c663dec5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,7 +67,7 @@ COPY docker/custom.conf docker/play.plugins /home/clowder/custom/ # Containers should NOT run as root as a good practice # numeric id to be compatible with openshift, will run as random userid:0 RUN mkdir -p /home/clowder/data && \ - chmod g+w /home/clowder/logs /home/clowder/data /home/clowder/custom + chmod 777 /home/clowder/logs /home/clowder/data /home/clowder/custom USER 10001 # command to run when starting docker diff --git a/app/api/Files.scala b/app/api/Files.scala index 7f141c4ec..6aba93af3 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -1678,6 +1678,7 @@ class Files @Inject()( val ds_spaces = ds.spaces for (ds_s <- ds_spaces) { spaces.decrementSpaceBytes(ds_s, file.length) + spaces.decrementFileCounter(ds_s, 1) } } diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala index d82ba2c5e..36894f105 100644 --- a/app/api/Spaces.scala +++ b/app/api/Spaces.scala @@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService, val userId = request.user.get.id val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, - datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) + datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) spaces.insert(c) match { case Some(id) => { appConfig.incrementCount('spaces, 1) diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala index 0f527f3db..0cbedeec3 100644 --- a/app/controllers/SecuredController.scala +++ b/app/controllers/SecuredController.scala @@ -153,7 +153,7 @@ trait SecuredController extends Controller { val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService]) spaces.get(id) match { case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user))) - case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user))) + case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user))) } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index c42b38fb7..885750209 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -167,8 +167,6 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS var creatorActual: User = null val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size)) val datasetsInSpace = datasets.listSpace(size, id.toString(), user) - val spaceBytes : Long = s.spaceBytes - val spaceFiles : Integer = getFilesPerSpace(id, user) val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user) val usersInSpace = spaces.getUsersInSpace(id, None) var curationObjectsInSpace: List[CurationObject] = List() @@ -217,7 +215,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => List.empty } sinkService.logSpaceViewEvent(s, user) - Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles)) + Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections)) } case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } @@ -414,7 +412,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS val newSpace = ProjectSpace(name = formData.name, description = formData.description, created = new Date, creator = userId, homePage = formData.homePage, logoURL = formData.logoURL, bannerURL = formData.bannerURL, - collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, + collectionCount = 0, datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled, status = formData.access, affiliatedSpaces = formData.affSpace) @@ -640,15 +638,4 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } } - - private def getFilesPerSpace(spaceId: UUID, user: Option[User]) : Integer = { - var spaceFiles: Integer = 0 - val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), user) - for (ds <- allDatasetsInSpace) { - val files_in_ds = ds.files.length - spaceFiles += files_in_ds - } - spaceFiles - } - } diff --git a/app/models/Space.scala b/app/models/Space.scala index 6f2d545e1..8b4edc823 100644 --- a/app/models/Space.scala +++ b/app/models/Space.scala @@ -22,6 +22,7 @@ case class ProjectSpace ( bannerURL: Option[URL], collectionCount: Integer, datasetCount: Integer, + fileCount: Integer, userCount: Integer, spaceBytes: Long, metadata: List[Metadata], diff --git a/app/services/SpaceService.scala b/app/services/SpaceService.scala index 3a9f1e60b..d7cf8e5f7 100644 --- a/app/services/SpaceService.scala +++ b/app/services/SpaceService.scala @@ -94,6 +94,10 @@ trait SpaceService { def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int) + def incrementFileCounter(space: UUID, increment: Long) + + def decrementFileCounter(space: UUID, decrement: Long) + def incrementSpaceBytes(space: UUID, increment: Long) def decrementSpaceBytes(space: UUID, decrement: Long) diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index dac16f0c1..2f456f7c1 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -1399,6 +1399,15 @@ class MongoDBDatasetService @Inject() ( } for (f <- dataset.files) { val notTheDataset = for (currDataset <- findByFileIdDirectlyContain(f) if !dataset.id.toString.equals(currDataset.id.toString)) yield currDataset + files.get(f) match { + case Some(file) => { + for(space <- dataset.spaces) { + spaces.decrementFileCounter(space, 1) + spaces.decrementSpaceBytes(space, file.length) + } + } + case None => Logger.error(s"Error file with with id ${f} no longer exists") + } if (notTheDataset.size == 0) files.removeFile(f, host, apiKey, user) } diff --git a/app/services/mongodb/MongoDBSpaceService.scala b/app/services/mongodb/MongoDBSpaceService.scala index 6f164f8d0..c7dd53292 100644 --- a/app/services/mongodb/MongoDBSpaceService.scala +++ b/app/services/mongodb/MongoDBSpaceService.scala @@ -375,7 +375,15 @@ class MongoDBSpaceService @Inject() ( } def decrementCollectionCounter(collection: UUID, space: UUID, decrement: Int): Unit = { - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) + } + + def incrementFileCounter(space: UUID, increment: Long ): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> increment), upsert=false, multi=false, WriteConcern.Safe) + } + + def decrementFileCounter(space: UUID, decrement: Long): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) } def incrementSpaceBytes(space: UUID, increment: Long ): Unit = { @@ -383,7 +391,7 @@ class MongoDBSpaceService @Inject() ( } def decrementSpaceBytes(space: UUID, decrement: Long): Unit = { - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe) } def removeCollection(collection:UUID, space:UUID): Unit = { @@ -398,11 +406,15 @@ class MongoDBSpaceService @Inject() ( */ def addDataset(dataset: UUID, space: UUID): Unit = { log.debug(s"Space Service - Adding $dataset to $space") - val datasetBytes = datasets.getBytesForDataset(dataset) - datasets.addToSpace(dataset, space) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) - + datasets.get(dataset) match { + case Some(x) => { + val datasetBytes = datasets.getBytesForDataset(dataset) + datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> x.files.length), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) + } + } } /** @@ -414,9 +426,15 @@ class MongoDBSpaceService @Inject() ( def removeDataset(dataset:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $dataset from $space") datasets.removeFromSpace(dataset, space) - val datasetBytes = datasets.getBytesForDataset(dataset) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe) - ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + datasets.get(dataset) match { + case Some(x) => { + val datasetBytes = datasets.getBytesForDataset(dataset) + datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * datasetBytes), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * x.files.length), upsert=false, multi=false, WriteConcern.Safe) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) + } + } } /** diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala index b5ede2229..3a625ae6c 100644 --- a/app/services/mongodb/MongoSalatPlugin.scala +++ b/app/services/mongodb/MongoSalatPlugin.scala @@ -452,6 +452,7 @@ class MongoSalatPlugin(app: Application) extends Plugin { // Adds space bytes to space updateMongo(updateKey = "update-space-bytes", updateSpaceBytes) + updateMongo(updateKey = "update-space-files", updateSpaceFiles) } private def updateMongo(updateKey: String, block: () => Unit): Unit = { @@ -513,7 +514,8 @@ class MongoSalatPlugin(app: Application) extends Plugin { val spacename = java.net.InetAddress.getLocalHost.getHostName val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"), homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata], - collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0) + collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, fileCount = 0, + spaceBytes = 0) ProjectSpaceDAO.save(newspace) val spaceId = new ObjectId(newspace.id.stringify) @@ -1707,4 +1709,16 @@ class MongoSalatPlugin(app: Application) extends Plugin { collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes)) } } + + private def updateSpaceFiles(): Unit = { + collection("spaces.projects").find().toList.foreach{ space => + var fileCount: Integer = 0 + val spaceId = space.get("_id") + val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList + spaceDatasets.foreach{ spaceDataset => + fileCount += spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty).length + } + collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("fileCount" -> fileCount)) + } + } } diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala index 949fa6dab..1d66d2f93 100644 --- a/app/util/FileUtils.scala +++ b/app/util/FileUtils.scala @@ -595,6 +595,7 @@ object FileUtils { val datasetSpaces = dataset.get.spaces for (s <- datasetSpaces){ spaceService.incrementSpaceBytes(s, file.length) + spaceService.incrementFileCounter(s, 1) } } } diff --git a/app/views/spaces/space.scala.html b/app/views/spaces/space.scala.html index 28ca667b2..696b81112 100644 --- a/app/views/spaces/space.scala.html +++ b/app/views/spaces/space.scala.html @@ -1,4 +1,4 @@ -@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String], spaceBytes: Long, spaceFiles: Integer)(implicit user: Option[models.User]) +@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String])(implicit user: Option[models.User]) @import play.api.libs.json._ @import play.api.Play.current @@ -119,10 +119,10 @@