diff --git a/pom.xml b/pom.xml index 3b0da97..7696289 100644 --- a/pom.xml +++ b/pom.xml @@ -5,13 +5,13 @@ org.scijava pom-scijava - 36.0.0 + 37.0.0 org.janelia.saalfeldlab n5-google-cloud - 4.0.1-SNAPSHOT + 4.1.0-SNAPSHOT N5 Google Cloud N5 library implementation using Google Cloud Storage backend. @@ -123,7 +123,7 @@ sign,deploy-to-scijava - 3.0.2 + 3.2.0 @@ -168,18 +168,35 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.janelia.saalfeldlab.n5.googlecloud.backend.** - - - - - + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.janelia.saalfeldlab.n5.googlecloud.backend.** + org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests.java + + + + + + + + run-backend-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + diff --git a/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudStorageURI.java b/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudStorageURI.java index 79cd428..e34b33e 100644 --- a/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudStorageURI.java +++ b/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudStorageURI.java @@ -42,6 +42,7 @@ public class GoogleCloudStorageURI private static final String storagePathPrefix = "/storage/v1/b/"; private static final String projectKey = "project"; + private final URI uri; private final String bucketName; private final String objectKey; private final String query; @@ -54,8 +55,9 @@ public GoogleCloudStorageURI( final String str ) public GoogleCloudStorageURI( final URI uri ) { + this.uri = uri; final String path; - if ( uri.getScheme().equalsIgnoreCase( "gs" ) ) + if ( uri.getScheme() != null && uri.getScheme().equalsIgnoreCase( "gs" ) ) { bucketName = uri.getAuthority(); objectKey = uri.getPath(); @@ -103,6 +105,10 @@ public String getBucket() return bucketName; } + public URI asURI() { + return this.uri; + } + public String getKey() { return objectKey; diff --git a/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudUtils.java b/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudUtils.java new file mode 100644 index 0000000..bbc2855 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/googlecloud/GoogleCloudUtils.java @@ -0,0 +1,52 @@ +package org.janelia.saalfeldlab.googlecloud; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.regex.Pattern; + +public class GoogleCloudUtils { + + public final static Pattern GS_SCHEME = Pattern.compile("gs", Pattern.CASE_INSENSITIVE); + public final static Pattern GS_HOST = Pattern.compile("(cloud\\.google|storage\\.googleapis)\\.com", Pattern.CASE_INSENSITIVE); + + private GoogleCloudUtils() { + + } + + public static String getGoogleCloudStorageKey(String uri) { + + return getGoogleCloudStorageKey(URI.create(uri)); + } + + public static String getGoogleCloudStorageKey(URI uri) { + + try { + // if key is null, return the empty string + final String key = new GoogleCloudStorageURI(uri).getKey(); + return key == null ? "" : key; + } catch (final Exception e) { + } + // parse key manually when GoogleCLoudStorageURI can't + final String path = uri.getPath().replaceFirst("^/", ""); + return path.substring(path.indexOf('/') + 1); + } + + public static Storage createGoogleCloudStorage(@Nullable final String googleCloudProjectId) { + + final GoogleCloudStorageClient storageClient = getGoogleCloudStorageClient(googleCloudProjectId); + if (storageClient == null) + return null; + + return storageClient.create(); + } + + public static GoogleCloudStorageClient getGoogleCloudStorageClient(@Nullable final String googleCloudProjectId) { + + return new GoogleCloudStorageClient(googleCloudProjectId != null ? googleCloudProjectId : + StorageOptions.getDefaultProjectId()); + + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/GoogleCloudStorageKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/GoogleCloudStorageKeyValueAccess.java index cd2461f..34c165f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/GoogleCloudStorageKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/GoogleCloudStorageKeyValueAccess.java @@ -9,6 +9,8 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobField; import com.google.cloud.storage.Storage.BlobListOption; +import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI; +import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; @@ -25,6 +27,8 @@ import java.nio.channels.Channels; import java.nio.channels.NonReadableChannelException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -34,21 +38,60 @@ public class GoogleCloudStorageKeyValueAccess implements KeyValueAccess { private final Storage storage; + private final GoogleCloudStorageURI containerURI; private final String bucketName; + protected static GoogleCloudStorageURI uncheckedContainerLocationStringToGoogleURI(final String uri) { + + try { + return new GoogleCloudStorageURI(uri); + } catch (Exception e) { + throw new N5Exception("Container location " + uri + " is an invalid URI", e); + } + } + + /** + * Creates a {@link KeyValueAccess} using a google cloud storage backend. + * + * @param storage the google cloud interface + * @param containerURI a string representation of a valid {@link URI } that points to the n5 container root. + * @param createBucket if true, a bucket will be created if it does not exist + * @throws N5Exception.N5IOException if the requested bucket does not exist and + * createBucket is false + */ + public GoogleCloudStorageKeyValueAccess(final Storage storage, final String containerURI, final boolean createBucket) throws N5Exception.N5IOException { + + this(storage, uncheckedContainerLocationStringToGoogleURI(containerURI), createBucket); + } + + /** + * Creates a {@link KeyValueAccess} using a google cloud storage backend. + * + * @param storage the google cloud interface + * @param containerURI the root of the n5 container root. + * @param createBucket if true, a bucket will be created if it does not exist + * @throws N5Exception.N5IOException if the requested bucket does not exist and + * createBucket is false + */ + public GoogleCloudStorageKeyValueAccess(final Storage storage, final URI containerURI, final boolean createBucket) throws N5Exception.N5IOException { + + this(storage, new GoogleCloudStorageURI(containerURI), createBucket); + } + /** * Creates a {@link KeyValueAccess} using a google cloud storage backend. - * + * * @param storage the google cloud interface - * @param bucketName the bucket name + * @param containerURI the root of the n5 container root. * @param createBucket if true, a bucket will be created if it does not exist * @throws N5Exception.N5IOException if the requested bucket does not exist and * createBucket is false */ - public GoogleCloudStorageKeyValueAccess(final Storage storage, final String bucketName, final boolean createBucket) throws N5Exception.N5IOException { + public GoogleCloudStorageKeyValueAccess(final Storage storage, final GoogleCloudStorageURI containerURI, final boolean createBucket) throws N5Exception.N5IOException { this.storage = storage; - this.bucketName = bucketName; + this.containerURI = containerURI; + this.bucketName = containerURI.getBucket(); if (!bucketExists(bucketName)) { if (createBucket) { @@ -61,6 +104,7 @@ public GoogleCloudStorageKeyValueAccess(final Storage storage, final String buck } private boolean bucketExists(final String bucketName) { + final Bucket bucket = storage.get(bucketName); return (bucket != null && bucket.exists()); } @@ -68,7 +112,10 @@ private boolean bucketExists(final String bucketName) { @Override public String[] components(final String path) { - return Arrays.stream(path.split("/")) + final String[] baseComponents = path.split("/"); + if (baseComponents.length <= 1) + return baseComponents; + return Arrays.stream(baseComponents) .filter(x -> !x.isEmpty()) .toArray(String[]::new); } @@ -86,11 +133,27 @@ public String compose(final String... components) { ); } + /** + * Compose a path from a base uri and subsequent components. + * + * @param uri the base path uri to resolve the components against + * @param components the components of the group path, relative to the n5 container + * @return the path + */ + @Override + public String compose(final URI uri, final String... components) { + + final String[] uriComponents = new String[components.length + 1]; + System.arraycopy(components, 0, uriComponents, 1, components.length); + uriComponents[0] = GoogleCloudUtils.getGoogleCloudStorageKey(uri); + return compose(uriComponents); + } + @Override public String parent(final String path) { final String[] components = components(path); - final String[] parentComponents =Arrays.copyOf(components, components.length - 1); + final String[] parentComponents = Arrays.copyOf(components, components.length - 1); return compose(parentComponents); } @@ -103,9 +166,9 @@ public String relativize(final String path, final String base) { * It's not true that the inputs are always referencing absolute paths, but it doesn't matter in this * case, since we only care about the relative portion of `path` to `base`, so the result always * ignores the absolute prefix anyway. */ - return normalize(uri("/" + base).relativize(uri("/" + path)).getPath()); + return GoogleCloudUtils.getGoogleCloudStorageKey(normalize(uri("/" + base).relativize(uri("/" + path)).getPath())); } catch (URISyntaxException e) { - throw new N5Exception("Cannot relativize path (" + path +") with base (" + base + ")", e); + throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e); } } @@ -115,11 +178,42 @@ public String normalize(final String path) { return N5URI.normalizeGroupPath(path); } + /** + * Create a URI that is the result of resolving the `normalPath` against the {@link #containerURI}. + * NOTE: {@link URI#resolve(URI)} always removes the last member of the receiver URIs path. + * That is undesirable behavior here, as we want to potentially keep the containerURI's + * full path, and just append `normalPath`. However, it's more complicated, as `normalPath` + * can also contain leading overlap with the trailing members of `containerURI.getPath()`. + * To properly resolve the two paths, we generate {@link Path}s from the results of {@link URI#getPath()} + * and use {@link Path#resolve(Path)}, which results in a guaranteed absolute path, with the + * desired path resolution behavior. That then is used to construct a new {@link URI}. + * Any query or fragment portions are ignored. Scheme and Authority are always + * inherited from {@link #containerURI}. + * + * @param normalPath EITHER a normalized path, or a valid URI + * @return the URI generated from resolving normalPath against containerURI + * @throws URISyntaxException if the given normal path is not a valid URI + */ @Override public URI uri(final String normalPath) throws URISyntaxException { - return N5URI.from( - "gs://" + bucketName + (normalPath.startsWith("/") ? normalPath : "/" + normalPath), null, null) - .getURI(); + + final URI asUri = containerURI.asURI(); + + if (normalize(normalPath).equals(normalize("/"))) + return asUri; + + final Path containerPath = Paths.get(asUri.getPath()); + final Path givenPath = Paths.get(URI.create(normalPath).getPath()); + + final Path resolvedPath = containerPath.resolve(givenPath); + final String[] pathParts = new String[resolvedPath.getNameCount() + 1]; + pathParts[0] = "/"; + for (int i = 0; i < resolvedPath.getNameCount(); i++) { + pathParts[i + 1] = resolvedPath.getName(i).toString(); + } + final String normalResolvedPath = compose(pathParts); + + return new URI(asUri.getScheme(), asUri.getAuthority(), normalResolvedPath, null, null); } @@ -130,7 +224,7 @@ public URI uri(final String normalPath) throws URISyntaxException { * either {@code path} or {@code path + "/"} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} exists, {@code false} otherwise */ @Override @@ -153,7 +247,7 @@ private boolean keyExists(final String key) { private static boolean blobExists(final Blob blob) { - return blob != null && blob .exists(); + return blob != null && blob.exists(); } private static String addTrailingSlash(final String path) { @@ -173,24 +267,24 @@ private static String removeLeadingSlash(final String path) { * leading "/", and then checks whether resulting {@code path} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} (with trailing "/") exists as a key, {@code false} otherwise */ @Override public boolean isDirectory(final String normalPath) { final String key = removeLeadingSlash(addTrailingSlash(normalPath)); - if (key.isEmpty() || keyExists(key)) - return true; - else { + if (key.equals(normalize("/"))) { + return bucketExists(bucketName); + } else { // not every directory will have a directly stored in the backend, // for example, if the container contents was copied to GCS with the cli - // in that case, check if any keys exist with the prefix, if so, its a directory + // in that case, check if any keys exist with the prefix, if so, it's a directory return storage.list(bucketName, - BlobListOption.prefix(key), - BlobListOption.pageSize(1), - BlobListOption.currentDirectory()) - .iterateAll().iterator().hasNext(); + BlobListOption.prefix(key), + BlobListOption.pageSize(1), + BlobListOption.currentDirectory()) + .iterateAll().iterator().hasNext(); } } @@ -201,7 +295,7 @@ public boolean isDirectory(final String normalPath) { * leading "/" and checks whether the resulting {@code path} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} exists as a key and has no trailing slash, {@code false} otherwise */ @Override @@ -226,7 +320,7 @@ public LockedChannel lockForWriting(final String normalPath) { * List all 'directory'-like children of a path. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return the array of child directories */ @Override @@ -248,9 +342,11 @@ private String[] list(final String normalPath, final boolean onlyDirectories) { BlobListOption.prefix(prefix), BlobListOption.currentDirectory(), BlobListOption.fields(BlobField.ID)); - for (final Iterator blobIterator = blobListing.iterateAll().iterator(); blobIterator.hasNext();) { + for (final Iterator blobIterator = blobListing.iterateAll().iterator(); blobIterator.hasNext(); ) { final Blob nextBlob = blobIterator.next(); final String blobName = nextBlob.getBlobId().getName(); + if (prefix.equals(blobName)) + continue; if (!onlyDirectories || blobName.endsWith("/")) { final String relativePath = relativize(blobName, prefix); if (!relativePath.isEmpty()) @@ -304,7 +400,7 @@ public void delete(final String normalPath) { while (page != null) { final BlobId[] ids = page.streamValues().map(Blob::getBlobId).toArray(BlobId[]::new); - if( ids.length > 0 ) // storage throws an error if ids is empty + if (ids.length > 0) // storage throws an error if ids is empty storage.delete(ids); page = page.getNextPage(); } @@ -320,8 +416,6 @@ public void delete(final String normalPath) { } } - - private class GoogleCloudObjectChannel implements LockedChannel { final String path; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageReader.java b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageReader.java index 4b38129..6dacf86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageReader.java @@ -6,28 +6,37 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import java.net.URISyntaxException; + +/* + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openReader()` or `N5KeyValueAccessReader` with + * an {@link GoogleCloudStorageKeyValueAccess} backend. + * */ +@Deprecated public class N5GoogleCloudStorageReader extends N5KeyValueReader { - /** - * TODO: reduce number of constructors ? - */ /** * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param basePath the base path relative to the bucket root - * @param gsonBuilder a GsonBuilder with custom configuration. - * @param cacheAttributes - * cache attribute and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openReader()` or `N5KeyValueAccessReader` with + * an {@link GoogleCloudStorageKeyValueAccess} backend. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param basePath the base path relative to the bucket root + * @param gsonBuilder a GsonBuilder with custom configuration. + * @param cacheAttributes cache attribute and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ + @Deprecated public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws N5Exception { super( @@ -36,23 +45,22 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName gsonBuilder, cacheAttributes); - if( !exists("/")) - throw new N5Exception.N5IOException("No container exists at " + basePath ); + if (!exists("/")) + throw new N5Exception.N5IOException("No container exists at " + basePath); } /** * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param basePath the base path relative to the bucket root - * @param cacheAttributes - * cache attribute and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param basePath the base path relative to the bucket root + * @param cacheAttributes cache attribute and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final String basePath, final boolean cacheAttributes) throws N5Exception { @@ -64,11 +72,11 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* Metadata are not cached. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param basePath the base path relative to the bucket root - * @param gsonBuilder a GsonBuilder with custom configuration. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param basePath the base path relative to the bucket root + * @param gsonBuilder a GsonBuilder with custom configuration. * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final String basePath, final GsonBuilder gsonBuilder) throws N5Exception { @@ -80,10 +88,10 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* Metadata are not cached. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param basePath the base path relative to the bucket root + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param basePath the base path relative to the bucket root * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final String basePath) throws N5Exception { @@ -95,17 +103,16 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* The n5 container root is the bucket's root. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param gsonBuilder a GsonBuilder with custom configuration. - * @param cacheAttributes - * cache attribute and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param gsonBuilder a GsonBuilder with custom configuration. + * @param cacheAttributes cache attribute and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws N5Exception { @@ -117,16 +124,15 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* The n5 container root is the bucket's root. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param cacheAttributes - * cache attribute and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param cacheAttributes cache attribute and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final boolean cacheAttributes) throws N5Exception { @@ -138,10 +144,10 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* The n5 container root is the bucket's root. Metadata are not cached. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name - * @param gsonBuilder a GsonBuilder with custom configuration. + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name + * @param gsonBuilder a GsonBuilder with custom configuration. * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName, final GsonBuilder gsonBuilder) throws N5Exception { @@ -153,9 +159,9 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName * Opens an {@link N5Writer} with a google cloud {@link Storage} storage backend. *

* The n5 container root is the bucket's root. Metadata are not cached. - * - * @param storage the google cloud storage instance - * @param bucketName the bucket name + * + * @param storage the google cloud storage instance + * @param bucketName the bucket name * @throws N5Exception if the reader could not be created */ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName) throws N5Exception { @@ -163,17 +169,16 @@ public N5GoogleCloudStorageReader(final Storage storage, final String bucketName this(storage, bucketName, "/", new GsonBuilder(), false); } - -// /** -// * Determines whether the current N5 container is stored at the root level of the bucket. -// * -// * @return -// */ -// protected boolean isContainerBucketRoot() { -// return isContainerBucketRoot(containerPath); -// } -// -// protected static boolean isContainerBucketRoot(String containerPath) { -// return removeLeadingSlash(containerPath).isEmpty(); -// } + // /** + // * Determines whether the current N5 container is stored at the root level of the bucket. + // * + // * @return + // */ + // protected boolean isContainerBucketRoot() { + // return isContainerBucketRoot(containerPath); + // } + // + // protected static boolean isContainerBucketRoot(String containerPath) { + // return removeLeadingSlash(containerPath).isEmpty(); + // } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageWriter.java index 048990c..5925795 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageWriter.java @@ -7,15 +7,22 @@ import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Reader; +/* + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openWriter()` or `N5KeyValueAccessWriter` with + * an {@link GoogleCloudStorageKeyValueAccess} backend. + * */ +@Deprecated public class N5GoogleCloudStorageWriter extends N5KeyValueWriter { - /** - * TODO: reduce number of constructors ? - */ /** * Opens an {@link N5Reader} with a google cloud {@link Storage} storage backend. - * + * + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openWriter()` or `N5KeyValueAccessWriter` with + * an {@link GoogleCloudStorageKeyValueAccess} backend. + * * @param storage the google cloud storage instance * @param bucketName the bucket name * @param basePath the base path relative to the bucket root @@ -29,6 +36,7 @@ public class N5GoogleCloudStorageWriter extends N5KeyValueWriter { * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ + @Deprecated public N5GoogleCloudStorageWriter(final Storage storage, final String bucketName, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws N5Exception { super( diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageBucketRootTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageBucketRootTest.java deleted file mode 100644 index f684565..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageBucketRootTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud; - - -import java.net.URI; -import java.net.URISyntaxException; - -import com.google.cloud.storage.Storage; - -public abstract class AbstractN5GoogleCloudStorageBucketRootTest extends AbstractN5GoogleCloudStorageTest { - - public AbstractN5GoogleCloudStorageBucketRootTest(final Storage storage) { - - super(storage); - } - - @Override - protected String tempN5Location() throws URISyntaxException { - return new URI("gs", tempBucketName(), "/", null).toString(); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageContainerPathTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageContainerPathTest.java deleted file mode 100644 index c430505..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageContainerPathTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud; - -import java.io.IOException; - -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import com.google.cloud.storage.Storage; - -public abstract class AbstractN5GoogleCloudStorageContainerPathTest extends AbstractN5GoogleCloudStorageTest { - - protected static String bucketName; - - public AbstractN5GoogleCloudStorageContainerPathTest(final Storage storage) { - - super(storage); - } - - @BeforeClass - public static void setup() throws IOException, URISyntaxException { - bucketName = tempBucketName(); - } - - @Override - protected String tempN5Location() throws URISyntaxException { - return new URI("gs", bucketName, tempContainerPath(), null).toString(); - } - - @AfterClass - public static void cleanup() throws IOException { - - storage.delete(bucketName); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageTest.java deleted file mode 100644 index 13d5829..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/AbstractN5GoogleCloudStorageTest.java +++ /dev/null @@ -1,201 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud; - -import com.google.gson.GsonBuilder; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.SecureRandom; - -import org.janelia.saalfeldlab.n5.AbstractN5Test; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.junit.Assert; -import org.junit.Test; - -import com.google.cloud.storage.Storage; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThrows; - -/** - * Base class for testing Google Cloud Storage N5 implementation. - * Tests that are specific to Google Cloud can be added here. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public abstract class AbstractN5GoogleCloudStorageTest extends AbstractN5Test { - - protected static Storage storage; - - public AbstractN5GoogleCloudStorageTest(final Storage storage) { - - AbstractN5GoogleCloudStorageTest.storage = storage; - } - - private static final SecureRandom random = new SecureRandom(); - - private static String generateName(String prefix, String suffix) { - - return prefix + Long.toUnsignedString(random.nextLong()) + suffix; - } - - protected static String tempBucketName() { - - return generateName("n5-test-", "-bucket"); - } - - protected static String tempContainerPath() { - - return generateName("/n5-test-", ".n5"); - } - - @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { - - final URI uri = new URI(tempN5Location()); - final String bucketName = uri.getHost(); - final String basePath = uri.getPath(); - return new N5GoogleCloudStorageWriter(storage, bucketName, basePath, new GsonBuilder()) { - - @Override public void close() { - - remove(); - super.close(); - } - }; - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final URI uri = new URI(location); - final String bucketName = uri.getHost(); - final String basePath = uri.getPath(); - return new N5GoogleCloudStorageWriter(storage, bucketName, basePath, gson); - } - - @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final URI uri = new URI(location); - final String bucketName = uri.getHost(); - final String basePath = uri.getPath(); - return new N5GoogleCloudStorageReader(storage, bucketName, basePath, gson); - } - - /** - * Currently, {@code N5GoogleCloudStorageReader#exists(String)} is implemented by listing objects under that group. - * This test case specifically tests its correctness. - * - * @throws IOException - */ - @Test - public void testExistsUsingListingObjects() throws IOException, URISyntaxException { - - try (N5Writer n5 = createN5Writer()) { - n5.createGroup("/one/two/three"); - - Assert.assertTrue(n5.exists("")); - Assert.assertTrue(n5.exists("/")); - - Assert.assertTrue(n5.exists("one")); - Assert.assertTrue(n5.exists("one/")); - Assert.assertTrue(n5.exists("/one")); - Assert.assertTrue(n5.exists("/one/")); - - Assert.assertTrue(n5.exists("one/two")); - Assert.assertTrue(n5.exists("one/two/")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one/two/")); - - Assert.assertTrue(n5.exists("one/two/three")); - Assert.assertTrue(n5.exists("one/two/three/")); - Assert.assertTrue(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two/three/")); - - Assert.assertFalse(n5.exists("one/tw")); - Assert.assertFalse(n5.exists("one/tw/")); - Assert.assertFalse(n5.exists("/one/tw")); - Assert.assertFalse(n5.exists("/one/tw/")); - - Assert.assertArrayEquals(new String[]{"one"}, n5.list("/")); - Assert.assertArrayEquals(new String[]{"two"}, n5.list("/one")); - Assert.assertArrayEquals(new String[]{"three"}, n5.list("/one/two")); - - Assert.assertArrayEquals(new String[]{}, n5.list("/one/two/three")); - assertThrows(N5Exception.N5IOException.class, () -> n5.list("/one/tw")); - - Assert.assertTrue(n5.remove("/one/two/three")); - Assert.assertFalse(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one")); - - Assert.assertTrue(n5.remove("/one")); - Assert.assertFalse(n5.exists("/one/two")); - Assert.assertFalse(n5.exists("/one")); - } - } - - @Override - @Test public void testReaderCreation() throws IOException, URISyntaxException { - - try (N5Writer writer = createN5Writer()) { - final String canonicalPath = writer.getURI().toString(); - - - final N5Reader n5r = createN5Reader(canonicalPath); - assertNotNull(n5r); - - // existing directory without attributes is okay; - // Remove and create to remove attributes store - writer.removeAttribute("/", "/"); - final N5Reader na = createN5Reader(canonicalPath); - assertNotNull(na); - - // existing location with attributes, but no version - writer.removeAttribute("/", "/"); - writer.setAttribute("/", "mystring", "ms"); - final N5Reader wa = createN5Reader(canonicalPath); - assertNotNull(wa); - - // existing directory with incompatible version should fail - writer.removeAttribute("/", "/"); - writer.setAttribute("/", N5Reader.VERSION_KEY, - new N5Reader.Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); - assertThrows("Incompatible version throws error", N5Exception.N5IOException.class, - () -> createN5Reader(canonicalPath)); - writer.remove(); - } - /* In the AbstractN5Test class, there is a final test to ensure the reader creation fails if the container doesn't exist. - * Unfortunately, the google cloud storage test framework doesn't support that during testing, - * so we cannot support it. If future cloud store testing frameworks support creating mock buckets, we can test then. */ - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageTests.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageTests.java new file mode 100644 index 0000000..fec276b --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/N5GoogleCloudStorageTests.java @@ -0,0 +1,191 @@ +/*- + * #%L + * N5 Google Cloud + * %% + * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5.googlecloud; + +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5URI; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Supplier; + +/** + * Base class for testing Google Cloud Storage N5 implementation. + * Tests that are specific to Google Cloud can be added here. + * + * @author Igor Pisarev <pisarevi@janelia.hhmi.org> + */ +@RunWith(Parameterized.class) +public class N5GoogleCloudStorageTests extends AbstractN5Test { + + + + public enum LocationInBucket { + ROOT(() -> "/", N5GoogleCloudStorageTests::tempBucketName), + KEY(N5GoogleCloudStorageTests::tempContainerPath, tempBucketName()::toString); + + public final Supplier getContainerPath; + private final Supplier getBucketName; + LocationInBucket(Supplier tempContainerPath, Supplier tempBucketaName) { + + this.getContainerPath = tempContainerPath; + this.getBucketName = tempBucketaName; + } + + String getPath() { + + return getContainerPath.get(); + } + + String getBucketName() { + + return getBucketName.get(); + } + } + + public enum UseCache { + CACHE(true), + NO_CACHE(false); + + final boolean cache; + + UseCache(boolean cache) { + + this.cache = cache; + } + } + + @Parameterized.Parameters(name = "Container at {0}, {1}") + public static Collection data() { + + return Arrays.asList(new Object[][]{ + {LocationInBucket.ROOT, UseCache.NO_CACHE}, + {LocationInBucket.ROOT, UseCache.CACHE}, + {LocationInBucket.KEY, UseCache.NO_CACHE}, + {LocationInBucket.KEY, UseCache.CACHE} + }); + } + + private static final SecureRandom random = new SecureRandom(); + + @Parameterized.Parameter() + public LocationInBucket containerLocation; + + @Parameterized.Parameter(1) + public UseCache useCache; + + protected static Storage lateinitStorage = null; + + @Parameterized.AfterParam() + public static void removeTestBuckets() { + + for (LocationInBucket location : LocationInBucket.values()) { + final String bucketName = location.getBucketName(); + try { + final GoogleCloudStorageKeyValueAccess kva = new GoogleCloudStorageKeyValueAccess(lateinitStorage, N5URI.encodeAsUri("gs://" + bucketName), false); + kva.delete(kva.normalize("/")); + } catch (Exception e) { + final Bucket bucket = lateinitStorage.get(bucketName);; + if (bucket == null || !bucket.exists()) + continue; + System.err.println("Exception After Tests, Could Not Delete Test Bucket:" + bucketName); + e.printStackTrace(); + } + } + } + private static String generateName(String prefix, String suffix) { + + return prefix + Long.toUnsignedString(random.nextLong()) + suffix; + } + + public static String tempBucketName() { + + return generateName("n5-test-", "-bucket"); + } + + protected static String tempContainerPath() { + + return generateName("/n5-test-", ".n5"); + } + + { + lateinitStorage = getGoogleCloudStorage(); + } + + protected Storage getGoogleCloudStorage() { + + return BackendGoogleCloudStorageFactory.getOrCreateStorage(); + } + + @Override protected String tempN5Location() throws URISyntaxException { + + final String containerPath = containerLocation.getPath(); + final String testBucket = containerLocation.getBucketName(); + return new URI("gs", testBucket, containerPath, null).toString(); + } + + @Override protected N5Writer createN5Writer() throws URISyntaxException, IOException { + + final String containerUri = tempN5Location(); + return createN5Writer(containerUri); + } + + + + @Override + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws URISyntaxException { + + final Storage storage = getGoogleCloudStorage(); + final KeyValueAccess kva = new GoogleCloudStorageKeyValueAccess(storage, N5URI.encodeAsUri(location), true); + return new N5KeyValueWriter(kva, location, gson, useCache.cache); + } + + @Override + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + + final Storage storage = getGoogleCloudStorage(); + final KeyValueAccess kva = new GoogleCloudStorageKeyValueAccess(storage, N5URI.encodeAsUri(location), false); + return new N5KeyValueReader(kva, location, gson, useCache.cache); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageBucketRootBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageBucketRootBackendTest.java deleted file mode 100644 index 2d61442..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageBucketRootBackendTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.backend; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageBucketRootTest; -import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageWriter; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using actual Google Cloud backend. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5GoogleCloudStorageBucketRootBackendTest extends AbstractN5GoogleCloudStorageBucketRootTest { - - public CachedN5GoogleCloudStorageBucketRootBackendTest() { - - super(BackendGoogleCloudStorageFactory.getOrCreateStorage()); - } - - @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { - - final URI uri = new URI(tempN5Location()); - final String bucketName = uri.getHost(); - final String basePath = uri.getPath(); - return new N5GoogleCloudStorageWriter(storage, bucketName, basePath, new GsonBuilder(), true) { - - @Override public void close() { - - remove(); - super.close(); - } - }; - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageContainerPathBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageContainerPathBackendTest.java deleted file mode 100644 index 4f170ba..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/CachedN5GoogleCloudStorageContainerPathBackendTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.backend; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageContainerPathTest; -import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageWriter; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using actual Google Cloud backend. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5GoogleCloudStorageContainerPathBackendTest extends AbstractN5GoogleCloudStorageContainerPathTest { - - public CachedN5GoogleCloudStorageContainerPathBackendTest() { - - super(BackendGoogleCloudStorageFactory.getOrCreateStorage()); - } - - @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { - - final URI uri = new URI(tempN5Location()); - final String bucketName = uri.getHost(); - final String basePath = uri.getPath(); - return new N5GoogleCloudStorageWriter(storage, bucketName, basePath, new GsonBuilder(), true) { - - @Override public void close() { - - remove(); - super.close(); - } - }; - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageBucketRootBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageBucketRootBackendTest.java deleted file mode 100644 index 25ac04b..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageBucketRootBackendTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.backend; - -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageBucketRootTest; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using actual Google Cloud backend. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5GoogleCloudStorageBucketRootBackendTest extends AbstractN5GoogleCloudStorageBucketRootTest { - - public N5GoogleCloudStorageBucketRootBackendTest() { - - super(BackendGoogleCloudStorageFactory.getOrCreateStorage()); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageContainerPathBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageContainerPathBackendTest.java deleted file mode 100644 index 5f8dfff..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/backend/N5GoogleCloudStorageContainerPathBackendTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.backend; - -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageContainerPathTest; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using actual Google Cloud backend. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5GoogleCloudStorageContainerPathBackendTest extends AbstractN5GoogleCloudStorageContainerPathTest { - - public N5GoogleCloudStorageContainerPathBackendTest() { - - super(BackendGoogleCloudStorageFactory.getOrCreateStorage()); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/MockGoogleCloudStorageFactory.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/MockGoogleCloudStorageFactory.java index 27425c0..8ae0231 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/MockGoogleCloudStorageFactory.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/MockGoogleCloudStorageFactory.java @@ -92,7 +92,7 @@ public Object create() { return storage; } - private static class MockBuckets implements Storage { + public static class MockBuckets implements Storage { final Storage delegate; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageBucketRootMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageBucketRootMockTest.java deleted file mode 100644 index 0c4a741..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageBucketRootMockTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.mock; - -import java.io.IOException; - -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageBucketRootTest; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using mock library. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5GoogleCloudStorageBucketRootMockTest extends AbstractN5GoogleCloudStorageBucketRootTest { - - public N5GoogleCloudStorageBucketRootMockTest() throws IOException { - - super(MockGoogleCloudStorageFactory.getOrCreateStorage()); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageContainerPathMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageContainerPathMockTest.java deleted file mode 100644 index bf721af..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageContainerPathMockTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/*- - * #%L - * N5 Google Cloud - * %% - * Copyright (C) 2017 - 2020 Igor Pisarev, Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.googlecloud.mock; - -import java.io.IOException; - -import org.janelia.saalfeldlab.n5.googlecloud.AbstractN5GoogleCloudStorageContainerPathTest; - -/** - * Initiates testing of the Google Cloud Storage N5 implementation using mock library. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5GoogleCloudStorageContainerPathMockTest extends AbstractN5GoogleCloudStorageContainerPathTest { - - public N5GoogleCloudStorageContainerPathMockTest() throws IOException { - - super(MockGoogleCloudStorageFactory.getOrCreateStorage()); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageMockTest.java new file mode 100644 index 0000000..3ec2d31 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/googlecloud/mock/N5GoogleCloudStorageMockTest.java @@ -0,0 +1,60 @@ +package org.janelia.saalfeldlab.n5.googlecloud.mock; + +import com.google.cloud.storage.Storage; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests; +import org.janelia.saalfeldlab.n5.googlecloud.mock.MockGoogleCloudStorageFactory; +import org.junit.Test; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +public class N5GoogleCloudStorageMockTest extends N5GoogleCloudStorageTests { + + @Override protected Storage getGoogleCloudStorage() { + + return MockGoogleCloudStorageFactory.getOrCreateStorage(); + } + + @Override + @Test public void testReaderCreation() throws IOException, URISyntaxException { + + try (N5Writer writer = createTempN5Writer()) { + final String canonicalPath = writer.getURI().toString(); + + final N5Reader n5r = createN5Reader(canonicalPath); + assertNotNull(n5r); + + // existing directory without attributes is okay; + // Remove and create to remove attributes store + writer.removeAttribute("/", "/"); + final N5Reader na = createN5Reader(canonicalPath); + assertNotNull(na); + + // existing location with attributes, but no version + writer.removeAttribute("/", "/"); + writer.setAttribute("/", "mystring", "ms"); + final N5Reader wa = createN5Reader(canonicalPath); + assertNotNull(wa); + + // existing directory with incompatible version should fail + writer.removeAttribute("/", "/"); + writer.setAttribute("/", N5Reader.VERSION_KEY, + new N5Reader.Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); + assertThrows("Incompatible version throws error", N5Exception.N5IOException.class, + () -> createN5Reader(canonicalPath)); + writer.remove(); + } + /* In the AbstractN5Test class, there is a final test to ensure the reader creation fails if the container doesn't exist. + * Unfortunately, the google cloud storage test framework doesn't support that during testing, + * so we cannot support it. If future cloud store testing frameworks support creating mock buckets, we can test then. */ + } +}