Merge pull request #2 from JaneliaSciComp/s3-support

S3 support
JaneliaSciComp · Jan 24, 2025 · 112fae3 · 112fae3
2 parents 7048c3e + 84fa4ab
commit 112fae3
Show file tree

Hide file tree

Showing 230 changed files with 7,582 additions and 9,198 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,9 +20,10 @@ build
 eclipseBuild
 node_modules
 npm-debug.log
-local
+mylocal
 target
 ear
 dist
 logs
-venv
+venv
+generated
diff --git a/README.md b/README.md
@@ -5,9 +5,9 @@
 
 ## General Notes
 
-JADE (JAnelia Data Environment) is a distributed storage system that can manage a set of configured volumes local to the machine or shared 
-- NFS-mounted from a remote host. Typically the shared volumes are read only with the exception of 
-the special OVERFLOW_VOLUME which is selected when a node runs out of available space on the local volumes.
+JADE (JAnelia Data Environment) is a distributed storage system that can manage a set of configured volumes local to
+the machine, shared - NFS-mounted from a remote host or S3 buckets. Typically the shared volumes are read only.
+
 The system consists of a master node (service) that manages the resource allocation and one or more workers or 
 agents that are responsible with reading and writing the data onto/from the managed volumes.
 

diff --git a/build.gradle b/build.gradle
@@ -3,46 +3,43 @@ allprojects {
     apply plugin: 'maven-publish'
     apply plugin: 'idea'
     group = 'org.janelia.jacs-storage'
-    version = '1.21.1.RELEASE'
+    version = '1.22.0.RC4'
 }
 
 subprojects {
 
+    sourceSets {
+        integrationTest {
+            compileClasspath += sourceSets.main.output + sourceSets.test.output
+            runtimeClasspath += sourceSets.main.output + sourceSets.test.output
+        }
+        perf
+    }
+
     configurations {
         integrationTestImplementation.extendsFrom(implementation, testImplementation)
-        integrationTestRuntimeOnly.extendsFrom(testRuntimeOnly)
+        integrationTestRuntimeOnly.extendsFrom(runtimeOnly, testRuntimeOnly)
 
         perfApi.extendsFrom(api)
         perfImplementation.extendsFrom(perfApi, implementation)
         perfRuntimeOnly.extendsFrom(runtimeOnly)
         perfRuntimeElements.extendsFrom(perfImplementation, perfRuntimeOnly)
     }
 
-    sourceSets {
-        perf {
-            compileClasspath += main.output
-            runtimeClasspath += main.output
-        }
-        integrationTest {
-            compileClasspath += main.output + test.output
-            runtimeClasspath += main.output + test.output
-        }
-    }
-
     apply from: "${rootDir}/deps.gradle"
 
     afterEvaluate {
         repositories {
             mavenLocal()
             mavenCentral()
             maven {
-                url "https://nexus.janelia.org/repository/maven-releases"
+                url = "https://nexus.janelia.org/repository/maven-releases"
             }
             maven {
-                url "https://nexus.janelia.org/repository/maven-snapshots"
+                url = "https://nexus.janelia.org/repository/maven-snapshots"
             }
             maven {
-                url "https://maven.scijava.org/content/groups/public"
+                url = "https://maven.scijava.org/content/groups/public"
             }
         }
 
@@ -107,10 +104,7 @@ subprojects {
         archiveClassifier.set('perf')
     }
 
-    tasks.register('integrationTest', Test) {
-        testClassesDirs += sourceSets.integrationTest.output.classesDirs
-        classpath = sourceSets.integrationTest.runtimeClasspath
-    }
+    tasks.register('integrationTest', Test)
 
     artifacts {
         perfRuntimeElements perfJar
@@ -143,21 +137,21 @@ subprojects {
                 def snapshotsRepoUrl = 'https://nexus.janelia.org/repository/maven-snapshots'
                 url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
                 credentials {
-                    username "${mavenRepoUser}"
-                    password "${mavenRepoPassword}"
+                    username = mavenRepoUser
+                    password = mavenRepoPassword
                 }
             }
         }
     }
 }
 
 tasks.register('updateWrapper', Wrapper) {
-    gradleVersion = "8.9"
+    gradleVersion = "8.10"
 }
 
 tasks.register('printVersion') {
     doLast {
-        println project.version
+        println(project.version)
     }
 }
 

diff --git a/deps.gradle b/deps.gradle
@@ -21,21 +21,23 @@ dependencies {
         CDI_SERVLET_LIB = "org.jboss.weld.servlet:weld-servlet-core:${WELD_VERSION}"
 
         // database drivers
-        MONGODB_VERSION = "3.10.2"
-        MONGODB_DRIVER_LIB = "org.mongodb:mongo-java-driver:${MONGODB_VERSION}"
+        MONGODB_VERSION = "5.1.2"
+        MONGODB_DRIVER_LIB = "org.mongodb:mongodb-driver-sync:${MONGODB_VERSION}"
+        MONGODB_BSON_LIB = "org.mongodb:bson:${MONGODB_VERSION}"
 
         // apache commons
-        COMMONS_LANG_VERSION = "3.9"
+        COMMONS_LANG_VERSION = "3.14.0"
         COMMONS_LANG_LIB = "org.apache.commons:commons-lang3:${COMMONS_LANG_VERSION}"
         COMMONS_COLLECTIONS_VERSION = "4.2"
         COMMONS_COLLECTIONS_LIB = "org.apache.commons:commons-collections4:${COMMONS_COLLECTIONS_VERSION}"
         COMMONS_COMPRESS_LIB_VERSION = "1.15"
         COMMONS_COMPRESS_LIB = "org.apache.commons:commons-compress:${COMMONS_COMPRESS_LIB_VERSION}"
         COMMONS_IO_LIB_VERSION = "2.11.0"
         COMMONS_IO_LIB = "commons-io:commons-io:${COMMONS_IO_LIB_VERSION}"
+        COMMONS_RNG_LIB = "org.apache.commons:commons-rng-simple:1.3"
 
         // Janelia libraries
-        JACS_MODEL_VERSION = "2.21.5"
+        JACS_MODEL_VERSION = "3.3.0"
         JACS_MODEL_RENDERING_LIB = "org.janelia.jacs-model:jacs-model-rendering:${JACS_MODEL_VERSION}"
 
         // JAX-RS API
@@ -74,7 +76,7 @@ dependencies {
         SWAGGER_JAXRS_LIB = "io.swagger:swagger-jersey2-jaxrs:${SWAGGER_VERSION}"
 
         // JSON
-        JACKSON_VERSION = "2.9.4"
+        JACKSON_VERSION = "2.9.10"
         JACKSON_CORE_LIB = "com.fasterxml.jackson.core:jackson-core:${JACKSON_VERSION}"
         JACKSON_ANNOTATIONS_LIB = "com.fasterxml.jackson.core:jackson-annotations:${JACKSON_VERSION}"
         JACKSON_DATABIND_LIB = "com.fasterxml.jackson.core:jackson-databind:${JACKSON_VERSION}"
@@ -96,8 +98,14 @@ dependencies {
         MESSAGEPACK_LIB = "org.msgpack:msgpack-core:0.8.14"
 
         // N5
-        N5_LIB = "org.janelia.saalfeldlab:n5:2.5.1"
-        N5_IMGLIB2_LIB = "org.janelia.saalfeldlab:n5-imglib2:4.3.0"
+        N5_LIB = "org.janelia.saalfeldlab:n5:3.3.0"
+        N5_UNIVERSE_LIB = "org.janelia.saalfeldlab:n5-universe:1.6.0"
+
+        // AWS SDK
+        AWS_SDK_VERSION = "2.30.5"
+        AWS_SDK_PLATFORM = "software.amazon.awssdk:bom:${AWS_SDK_VERSION}"
+        AWS_CRT_LIB = "software.amazon.awssdk.crt:aws-crt:0.33.9"
+        AWS_S3_LIB = "software.amazon.awssdk:s3"
 
         // testing
         JUNIT_VERSION = "4.13.1"
@@ -114,10 +122,9 @@ dependencies {
         POWERMOCK_API_MOCKITO_LIB = "org.powermock:powermock-api-mockito2:${POWERMOCKITO_VERSION}"
 
         // benchmarks
-        BENCHMARK_VERSION = "1.21"
+        BENCHMARK_VERSION = "1.37"
         BENCHMARK_CORE_LIB = "org.openjdk.jmh:jmh-core:${BENCHMARK_VERSION}"
         BENCHMARK_ANN_PROCESSOR_LIB = "org.openjdk.jmh:jmh-generator-annprocess:${BENCHMARK_VERSION}"
-
     }
 
     // Included in the implementation of ALL projects

diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/jacsstorage-agentweb/build.gradle b/jacsstorage-agentweb/build.gradle
@@ -25,8 +25,7 @@ project(":jacsstorage-agentweb") {
                        JAX_RS_JERSEY_CDI_SERVLET_LIB,
                        JAX_RS_JERSEY_INJECT_LIB,
                        JAX_RS_JERSEY_MEDIA_JSON_LIB,
-                       N5_LIB,
-                       N5_IMGLIB2_LIB
+                       N5_UNIVERSE_LIB
 
         testImplementation JAX_RS_JERSEY_TEST_LIB,
                            JAX_RS_JERSEY_TEST_PROVIDER_LIB
@@ -35,6 +34,7 @@ project(":jacsstorage-agentweb") {
 
         perfImplementation project,
                            project(path: ":jacsstorage-services", configuration: "perfRuntimeElements"),
+                           COMMONS_RNG_LIB,
                            JAX_RS_JERSEY_TEST_LIB,
                            JAX_RS_JERSEY_TEST_PROVIDER_LIB
     }
@@ -126,16 +126,16 @@ project(":jacsstorage-agentweb") {
             }
         }
 
-        from ("${buildDir}/install/${project.name}/bin") {
+        from ("${layout.buildDirectory}/install/${project.name}/bin") {
             into "bin"
             filter(org.apache.tools.ant.filters.ExpandProperties, project: filteringProperties(installEnv))
         }
 
-        from ("${buildDir}/install/${project.name}/lib") {
+        from ("${layout.buildDirectory}/install/${project.name}/lib") {
             into "lib"
         }
 
-        from ("${buildDir}/install/${project.name}/swagger-webapp") {
+        from ("${layout.buildDirectory}/install/${project.name}/swagger-webapp") {
             into "swagger-webapp"
         }
 

diff --git a/jacsstorage-agentweb/src/main/java/org/janelia/jacsstorage/app/JAXAgentStorageApp.java b/jacsstorage-agentweb/src/main/java/org/janelia/jacsstorage/app/JAXAgentStorageApp.java
@@ -11,12 +11,13 @@ public class JAXAgentStorageApp extends AbstractJAXApp {
     protected Set<Class<?>> getAppClasses() {
         return ImmutableSet.of(
                 AgentConnectionResource.class,
-                AgentStorageResource.class,
+                DataBundleStorageResource.class,
                 PathBasedAgentStorageResource.class,
                 VolumeStorageResource.class,
                 VolumeQuotaResource.class,
                 AgentWebdavResource.class,
-                N5StorageResource.class
+                N5StorageResource.class,
+                ContentNotFoundRequestHandler.class
         );
     }
 }
diff --git a/jacsstorage-agentweb/src/main/java/org/janelia/jacsstorage/app/JacsAgentStorageApp.java b/jacsstorage-agentweb/src/main/java/org/janelia/jacsstorage/app/JacsAgentStorageApp.java
@@ -14,7 +14,7 @@
 import org.janelia.jacsstorage.cdi.qualifier.ApplicationProperties;
 import org.janelia.jacsstorage.config.ApplicationConfig;
 import org.janelia.jacsstorage.coreutils.NetUtils;
-import org.janelia.jacsstorage.service.localservice.StorageVolumeBootstrapper;
+import org.janelia.jacsstorage.service.impl.localservice.StorageVolumeBootstrapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

diff --git a/...rage/rest/ContentFilterRequestHelper.java → ...sthelpers/ContentAccessRequestHelper.java b/...rage/rest/ContentFilterRequestHelper.java → ...sthelpers/ContentAccessRequestHelper.java
@@ -1,15 +1,15 @@
-package org.janelia.jacsstorage.rest;
+package org.janelia.jacsstorage.requesthelpers;
+
+import javax.ws.rs.core.MultivaluedMap;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.janelia.jacsstorage.io.ContentFilterParams;
+import org.janelia.jacsstorage.service.ContentAccessParams;
 
-import javax.ws.rs.core.MultivaluedMap;
+public class ContentAccessRequestHelper {
 
-class ContentFilterRequestHelper {
-
-    static ContentFilterParams createContentFilterParamsFromQuery(MultivaluedMap<String, String> queryParameters) {
-        ContentFilterParams filterParams = new ContentFilterParams();
+    public static ContentAccessParams createContentAccessParamsFromQuery(MultivaluedMap<String, String> queryParameters) {
+        ContentAccessParams filterParams = new ContentAccessParams();
         queryParameters.forEach((k, vs) -> {
             if ("filterType".equalsIgnoreCase(k)) {
                 if (CollectionUtils.isNotEmpty(vs))
@@ -21,9 +21,6 @@ static ContentFilterParams createContentFilterParamsFromQuery(MultivaluedMap<Str
                 if (CollectionUtils.isNotEmpty(vs)) {
                     filterParams.setMaxDepth(vs.stream().filter(StringUtils::isNotBlank).map(s -> Integer.valueOf(s)).findFirst().orElse(1));
                 }
-            } else if ("useNaturalSort".equalsIgnoreCase(k)) {
-                if (CollectionUtils.isNotEmpty(vs))
-                    filterParams.setNaturalSort(Boolean.parseBoolean(vs.get(0)));
             } else if ("noSize".equalsIgnoreCase(k)) {
                 if (CollectionUtils.isNotEmpty(vs))
                     filterParams.setEstimateSizeDisabled(Boolean.parseBoolean(vs.get(0)));

diff --git a/...ge-agentweb/src/main/java/org/janelia/jacsstorage/rest/ContentNotFoundRequestHandler.java b/...ge-agentweb/src/main/java/org/janelia/jacsstorage/rest/ContentNotFoundRequestHandler.java
@@ -0,0 +1,34 @@
+package org.janelia.jacsstorage.rest;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+import org.apache.commons.lang3.StringUtils;
+import org.janelia.jacsstorage.service.NoContentFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Provider
+public class ContentNotFoundRequestHandler implements ExceptionMapper<NoContentFoundException> {
+    private static final Logger LOG = LoggerFactory.getLogger(ContentNotFoundRequestHandler.class);
+
+    @Context
+    private HttpServletRequest request;
+
+    @Override
+    public Response toResponse(NoContentFoundException exception) {
+        Response.ResponseBuilder responseBuilder = Response
+                .status(Response.Status.NOT_FOUND);
+        String errorMessage = StringUtils.defaultIfBlank(exception.getMessage(), "No content found");
+        responseBuilder
+                .entity(new ErrorResponse(errorMessage))
+                .type(MediaType.APPLICATION_JSON);
+        return responseBuilder.build();
+    }
+
+}