Wip/codecs shards

README.md

@@ -22,7 +22,7 @@ N5 group is not a single file but simply a directory on the file system.  Meta-d
 
 1. All directories of the file system are N5 groups.
 2. A JSON file `attributes.json` in a directory contains arbitrary attributes.  A group without attributes may not have an `attributes.json` file.
-3. The version of this specification is 4.0.0 and is stored in the "n5" attribute of the root group "/".
+3. The version of this specification is 1.0.0 and is stored in the "n5" attribute of the root group "/".
 4. A dataset is a group with the mandatory attributes:
    * dimensions (e.g. [100, 200, 300]),
    * blockSize (e.g. [64, 64, 64]),
@@ -38,7 +38,7 @@ N5 group is not a single file but simply a directory on the file system.  Meta-d
      * xz with parameters
        * preset (integer, default 6).
 
-   Custom compression schemes with arbitrary parameters can be added using [compression annotations](#extensible-compression-schemes), e.g. [N5 Blosc](https://github.com/saalfeldlab/n5-blosc) and [N5 ZStandard](https://github.com/JaneliaSciComp/n5-zstandard/).
+   Custom compression schemes with arbitrary parameters can be added using [compression annotations](#extensible-compression-schemes), e.g. [N5 Blosc](https://github.com/saalfeldlab/n5-blosc).
 5. Chunks are stored in a directory hierarchy that enumerates their positive integer position in the chunk grid (e.g. `0/4/1/7` for chunk grid position p=(0, 4, 1, 7)).
 6. Datasets are sparse, i.e. there is no guarantee that all chunks of a dataset exist.
 7. Chunks cannot be larger than 2GB (2<sup>31</sup>Bytes).
@@ -134,3 +134,4 @@ Custom compression schemes can be implemented using the annotation discovery mec
 HDF5 is a great format that provides a wealth of conveniences that I do not want to miss.  It's inefficiency for parallel writing, however, limit its applicability for handling of very large n-dimensional data.
 
 N5 uses the native filesystem of the target platform and JSON files to specify basic and custom meta-data as attributes.  It aims at preserving the convenience of HDF5 where possible but doesn't try too hard to be a full replacement.
+Please do not take this project too seriously, we will see where it will get us and report back when more data is available.

pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.scijava</groupId>
 		<artifactId>pom-scijava</artifactId>
-		<version>38.0.1</version>
+		<version>40.0.0</version>
 		<relativePath />
 	</parent>
 
@@ -161,13 +161,36 @@
 			<groupId>com.google.code.gson</groupId>
 			<artifactId>gson</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.scijava</groupId>
+			<artifactId>scijava-common</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-compress</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+		</dependency>
 
 		<!-- Test dependencies -->
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.janelia.saalfeldlab</groupId>
+			<artifactId>n5-universe</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>org.janelia.saalfeldlab</groupId>
+					<artifactId>n5</artifactId>
+				</exclusion>
+			</exclusions>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>net.imagej</groupId>
 			<artifactId>ij</artifactId>
@@ -194,14 +217,6 @@
 			<version>${commons-collections4.version}</version>
 			<scope>test</scope>
 		</dependency>
-		<dependency>
-			<groupId>org.scijava</groupId>
-			<artifactId>scijava-common</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-compress</artifactId>
-		</dependency>
 	</dependencies>
 
 	<repositories>

src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java

@@ -25,6 +25,11 @@
  */
 package org.janelia.saalfeldlab.n5;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
 /**
  * Abstract base class for {@link DataBlock} implementations.
  *
@@ -63,4 +68,20 @@ public T getData() {
 
 		return data;
 	}
+
+	@Override
+	public void readData(final DataInput input) throws IOException {
+
+		final ByteBuffer buffer = toByteBuffer();
+		input.readFully(buffer.array());
+		readData(buffer);
+	}
+
+	@Override
+	public void writeData(final DataOutput output) throws IOException {
+
+		final ByteBuffer buffer = toByteBuffer();
+		output.write(buffer.array());
+	}
+
 }

src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java

@@ -0,0 +1,11 @@
+package org.janelia.saalfeldlab.n5;
+
+public interface BlockParameters {
+
+	public long[] getDimensions();
+
+	public int getNumDimensions();
+
+	public int[] getBlockSize();
+
+}

src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java

@@ -25,6 +25,8 @@
  */
 package org.janelia.saalfeldlab.n5;
 
+import java.io.DataInput;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 
 public class ByteArrayDataBlock extends AbstractDataBlock<byte[]> {
@@ -47,6 +49,12 @@ public void readData(final ByteBuffer buffer) {
 			buffer.get(getData());
 	}
 
+	@Override
+	public void readData(final DataInput inputStream) throws IOException {
+
+		inputStream.readFully(data);
+	}
+
 	@Override
 	public int getNumElements() {
 

src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java

@@ -32,8 +32,10 @@
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.janelia.saalfeldlab.n5.Compression.CompressionType;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
 
 @CompressionType("bzip2")
+@NameConfig.Name("bzip2")
 public class Bzip2Compression implements DefaultBlockReader, DefaultBlockWriter, Compression {
 
 	private static final long serialVersionUID = -4873117458390529118L;
@@ -52,17 +54,29 @@ public Bzip2Compression() {
 	}
 
 	@Override
-	public InputStream getInputStream(final InputStream in) throws IOException {
+	public InputStream decode(final InputStream in) throws IOException {
 
 		return new BZip2CompressorInputStream(in);
 	}
 
 	@Override
-	public OutputStream getOutputStream(final OutputStream out) throws IOException {
+	public InputStream getInputStream(final InputStream in) throws IOException {
+
+		return decode(in);
+	}
+
+	@Override
+	public OutputStream encode(final OutputStream out) throws IOException {
 
 		return new BZip2CompressorOutputStream(out, blockSize);
 	}
 
+	@Override
+	public OutputStream getOutputStream(final OutputStream out) throws IOException {
+
+		return encode(out);
+	}
+
 	@Override
 	public Bzip2Compression getReader() {
 

src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java

@@ -27,14 +27,14 @@
 
 import java.lang.reflect.Type;
 
-import com.google.gson.JsonSyntaxException;
 import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
 import org.janelia.saalfeldlab.n5.cache.N5JsonCache;
 import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer;
 
 import com.google.gson.Gson;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import com.google.gson.JsonSyntaxException;
 
 /**
  * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON
@@ -70,7 +70,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) {
 			return null;
 
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPath, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath);
 		}
@@ -96,7 +96,7 @@ default <T> T getAttribute(
 
 		final JsonElement attributes;
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPathName, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
 		}
@@ -117,7 +117,7 @@ default <T> T getAttribute(
 		final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
 		JsonElement attributes;
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPathName, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
 		}
@@ -133,7 +133,7 @@ default boolean exists(final String pathName) {
 
 		final String normalPathName = N5URI.normalizeGroupPath(pathName);
 		if (cacheMeta())
-			return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().isGroup(normalPathName, getAttributesKey());
 		else {
 			return existsFromContainer(normalPathName, null);
 		}
@@ -176,7 +176,7 @@ default boolean datasetExists(final String pathName) throws N5IOException {
 
 		final String normalPathName = N5URI.normalizeGroupPath(pathName);
 		if (cacheMeta()) {
-			return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().isDataset(normalPathName, getAttributesKey());
 		}
 		return isDatasetFromContainer(normalPathName);
 	}
@@ -208,7 +208,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException {
 
 		/* If cached, return the cache */
 		if (cacheMeta()) {
-			return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().getAttributes(groupPath, getAttributesKey());
 		} else {
 			return GsonKeyValueN5Reader.super.getAttributes(groupPath);
 		}

src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java

@@ -59,9 +59,9 @@ default void createGroup(final String path) throws N5Exception {
 		// else if exists is true (then a dataset is present) so throw an exception to avoid
 		// overwriting / invalidating existing data
 		if (cacheMeta()) {
-			if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON))
+			if (getCache().isGroup(normalPath, getAttributesKey()))
 				return;
-			else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
+			else if (getCache().exists(normalPath, getAttributesKey())) {
 				throw new N5Exception("Can't make a group on existing path.");
 			}
 		}
@@ -88,8 +88,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
 			for (final String child : pathParts) {
 
 				final String childPath = parent.isEmpty() ? child : parent + "/" + child;
-				getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
-				getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
+				getCache().initializeNonemptyCache(childPath, getAttributesKey());
+				getCache().updateCacheInfo(childPath, getAttributesKey());
 
 				// only add if the parent exists and has children cached already
 				if (parent != null && !child.isEmpty())
@@ -130,7 +130,7 @@ default void writeAndCacheAttributes(
 				nullRespectingAttributes = getGson().toJsonTree(attributes);
 			}
 			/* Update the cache, and write to the writer */
-			getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes);
+			getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes);
 		}
 	}
 

src/main/java/org/janelia/saalfeldlab/n5/Compression.java

@@ -25,21 +25,29 @@
  */
 package org.janelia.saalfeldlab.n5;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
+import org.janelia.saalfeldlab.n5.codec.Codec;
 import org.scijava.annotations.Indexable;
 
 /**
+ * Deprecated: {@link Compression}s are no longer a special case.
+ * <br>
+ * Use {@link Codec.BytesCodec} for implementing compressors
+ * <p> </p>
  * Compression scheme interface.
  *
  * @author Stephan Saalfeld
  */
-public interface Compression extends Serializable {
+public interface Compression extends Serializable, Codec.BytesCodec {
 
 	/**
 	 * Annotation for runtime discovery of compression schemes.
@@ -49,7 +57,7 @@ public interface Compression extends Serializable {
 	@Inherited
 	@Target(ElementType.TYPE)
 	@Indexable
-	public static @interface CompressionType {
+	@interface CompressionType {
 
 		String value();
 	}
@@ -61,9 +69,10 @@ public interface Compression extends Serializable {
 	@Retention(RetentionPolicy.RUNTIME)
 	@Inherited
 	@Target(ElementType.FIELD)
-	public static @interface CompressionParameter {}
+	@interface CompressionParameter {}
 
-	public default String getType() {
+	@Override
+	default String getType() {
 
 		final CompressionType compressionType = getClass().getAnnotation(CompressionType.class);
 		if (compressionType == null)
@@ -72,7 +81,29 @@ public default String getType() {
 			return compressionType.value();
 	}
 
-	public BlockReader getReader();
 
-	public BlockWriter getWriter();
+	BlockReader getReader();
+
+	BlockWriter getWriter();
+
+	/**
+	 * Decode an {@link InputStream}.
+	 *
+	 * @param in
+	 *            input stream
+	 * @return the decoded input stream
+	 */
+	@Override
+	InputStream decode(InputStream in) throws IOException;
+
+	/**
+	 * Encode an {@link OutputStream}.
+	 *
+	 * @param out
+	 *            the output stream
+	 * @return the encoded output stream
+	 */
+	@Override
+	OutputStream encode(OutputStream out) throws IOException;
+
 }