Skip to content

Commit

Permalink
fix/test: add writeShardTest
Browse files Browse the repository at this point in the history
* fix ShardIndexBuilder
  • Loading branch information
bogovicj committed Jan 2, 2025
1 parent 1c9e601 commit b62dc00
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,7 @@ public ShardIndexBuilder indexLocation(IndexLocation location) {

this.location = location;
this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location);

if (location == IndexLocation.END)
currentOffset = 0;
else
currentOffset = temporaryIndex.numBytes();

updateInitialOffset();
return this;
}

Expand All @@ -53,8 +48,9 @@ public IndexLocation getLocation() {
public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) {

this.codecs = codecs;
final ShardIndex newIndex = new ShardIndex(temporaryIndex.getSize(), temporaryIndex.getLocation(), codecs);
final ShardIndex newIndex = new ShardIndex(shard.getBlockGridSize(), temporaryIndex.getLocation(), codecs);
this.temporaryIndex = newIndex;
updateInitialOffset();
return this;
}

Expand All @@ -78,4 +74,13 @@ public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) {
return this;
}

private void updateInitialOffset() {

if (location == IndexLocation.END)
currentOffset = 0;
else
currentOffset = temporaryIndex.numBytes();

}

}
58 changes: 55 additions & 3 deletions src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package org.janelia.saalfeldlab.n5.shard;

import com.google.gson.GsonBuilder;
import org.janelia.saalfeldlab.n5.Bzip2Compression;
import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
import org.janelia.saalfeldlab.n5.GzipCompression;
import org.janelia.saalfeldlab.n5.Lz4Compression;
import org.janelia.saalfeldlab.n5.N5Writer;
import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes;
import org.janelia.saalfeldlab.n5.XzCompression;
import org.janelia.saalfeldlab.n5.codec.BytesCodec;
import org.janelia.saalfeldlab.n5.codec.Codec;
import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec;
Expand Down Expand Up @@ -150,4 +147,59 @@ public void writeReadBlockTest() {
}
}

@Test
public void writeReadShardTest() {

final N5Factory factory = new N5Factory();
final GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
factory.gsonBuilder(gsonBuilder);
factory.cacheAttributes(false);

final N5Writer writer = factory.openWriter("src/test/resources/shardExamples/test.n5");

final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes(
new long[]{4, 4},
new int[]{4, 4},
new int[]{2, 2},
DataType.UINT8,
new Codec[]{new N5BlockCodec(dataByteOrder)},
new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()},
indexLocation
);
writer.createDataset("wholeShard", datasetAttributes);
writer.deleteBlock("wholeShard", 0, 0);

final int[] blockSize = datasetAttributes.getBlockSize();
final DataType dataType = datasetAttributes.getDataType();
final int numElements = 2 * 2;

final HashMap<long[], byte[]> writtenBlocks = new HashMap<>();

final InMemoryShard<byte[]> shard = new InMemoryShard<byte[]>(datasetAttributes, new long[]{0, 0});

for (int idx1 = 1; idx1 >= 0; idx1--) {
for (int idx2 = 1; idx2 >= 0; idx2--) {
final long[] gridPosition = {idx1, idx2};
final DataBlock<byte[]> dataBlock = (DataBlock<byte[]>)dataType.createDataBlock(blockSize, gridPosition, numElements);
byte[] data = dataBlock.getData();
for (int i = 0; i < data.length; i++) {
data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i);
}

shard.addBlock(dataBlock);
writtenBlocks.put(gridPosition, data);
}
}

writer.writeShard("wholeShard", datasetAttributes, shard);

for (Map.Entry<long[], byte[]> entry : writtenBlocks.entrySet()) {
final long[] otherGridPosition = entry.getKey();
final byte[] otherData = entry.getValue();
final DataBlock<byte[]> otherBlock = (DataBlock<byte[]>)writer.readBlock("wholeShard", datasetAttributes, otherGridPosition);
Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherBlock.getData());
}
}

}

0 comments on commit b62dc00

Please sign in to comment.