Skip to content

Commit

Permalink
Merge branch 'dev' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
aschampion authored Dec 6, 2022
2 parents 3e3a3d2 + 6b924d9 commit a5664a6
Show file tree
Hide file tree
Showing 24 changed files with 719 additions and 512 deletions.
17 changes: 17 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: CI
on: [push, pull_request]

jobs:
test:
uses: aschampion/gh-actions/.github/workflows/rust-test.yml@v0
with:
msrv: 1.56

semver-checks:
uses: aschampion/gh-actions/.github/workflows/rust-semver-checks.yml@v0

publish:
uses: aschampion/gh-actions/.github/workflows/rust-publish.yml@v0
needs: [test]
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/v')
secrets: inherit
1 change: 1 addition & 0 deletions .rustfmt.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
imports_layout = "Vertical"
36 changes: 0 additions & 36 deletions .travis.yml

This file was deleted.

25 changes: 25 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).

<!-- next-header -->

## [Unreleased]
### Changed
- Increase MSRV 1.39 -> 1.56 for tool and dependency upgrades.

## [0.7.6] - 2020-10-26
### Added
Expand Down Expand Up @@ -180,3 +185,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.


## [0.1.0] - 2018-02-28


<!-- next-url -->
[Unreleased]: https://github.com/aschampion/rust-n5/compare/0.7.6...HEAD
[0.7.6]: https://github.com/aschampion/rust-n5/compare/0.7.5...0.7.6
[0.7.5]: https://github.com/aschampion/rust-n5/compare/0.7.4...0.7.5
[0.7.4]: https://github.com/aschampion/rust-n5/compare/0.7.3...0.7.4
[0.7.3]: https://github.com/aschampion/rust-n5/compare/0.7.2...0.7.3
[0.7.2]: https://github.com/aschampion/rust-n5/compare/0.7.1...0.7.2
[0.7.1]: https://github.com/aschampion/rust-n5/compare/0.7.0...0.7.1
[0.7.0]: https://github.com/aschampion/rust-n5/compare/0.6.1...0.7.0
[0.6.1]: https://github.com/aschampion/rust-n5/compare/0.6.0...0.6.1
[0.6.0]: https://github.com/aschampion/rust-n5/compare/0.5.0...0.6.0
[0.5.0]: https://github.com/aschampion/rust-n5/compare/0.4.0...0.5.0
[0.4.0]: https://github.com/aschampion/rust-n5/compare/0.3.0...0.4.0
[0.3.0]: https://github.com/aschampion/rust-n5/compare/0.2.3...0.3.0
[0.2.3]: https://github.com/aschampion/rust-n5/compare/0.2.2...0.2.3
[0.2.2]: https://github.com/aschampion/rust-n5/compare/0.2.1...0.2.2
[0.2.1]: https://github.com/aschampion/rust-n5/compare/0.2.0...0.2.1
[0.2.0]: https://github.com/aschampion/rust-n5/compare/0.1.0...0.2.0
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
name = "n5"
version = "0.7.6"
edition = "2018"
rust-version = "1.39"
license = "MIT/Apache-2.0"
authors = ["Andrew Champion <[email protected]>"]
description = "Rust implementation of the N5 tensor file system format"
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# N5 [![Build Status](https://travis-ci.org/aschampion/rust-n5.svg?branch=master)](https://travis-ci.org/aschampion/rust-n5) [![Coverage](https://codecov.io/gh/aschampion/rust-n5/branch/master/graph/badge.svg)](https://codecov.io/gh/aschampion/rust-n5)
# N [![Build Status](https://github.com/aschampion/rust-n5/actions/workflows/ci.yml/badge.svg)](https://github.com/aschampion/rust-n5/actions/workflows/ci.yml/) [![Coverage](https://codecov.io/gh/aschampion/rust-n5/branch/master/graph/badge.svg)](https://codecov.io/gh/aschampion/rust-n5)

A (mostly pure) rust implementation of the [N5 "Not HDF5" n-dimensional tensor file system storage format](https://github.com/saalfeldlab/n5) created by the Saalfeld lab at Janelia Research Campus.

Expand All @@ -10,7 +10,7 @@ Compatible with Java N5 Version 2.1.3.

## Minimum supported Rust version (MSRV)

Stable 1.39
Stable 1.56

## Quick start

Expand Down
65 changes: 32 additions & 33 deletions benches/parallel_write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@
//! will take several hours to run.
#![feature(test)]


extern crate test;


use std::fs::File;
use std::io::BufReader;

Expand All @@ -38,7 +36,6 @@ use tiff::decoder::{
use n5::prelude::*;
use n5::smallvec::smallvec;


lazy_static! {
static ref TEST_IMAGE: Vec<i8> = {
let mut pixels = Vec::with_capacity(163 * 163 * 93);
Expand All @@ -54,7 +51,7 @@ lazy_static! {
for p in img {
pixels.push(p as i8);
}
},
}
}

decoder.next_image().unwrap();
Expand All @@ -66,15 +63,12 @@ lazy_static! {
const BLOCK_DIM: u32 = 64;
const N_BLOCKS: u64 = 5;

fn write<T, N5>(
n: &N5,
compression: &CompressionType,
block_data: &[T],
pool_size: usize,
) where T: 'static + std::fmt::Debug + ReflectedType + PartialEq + Default + Sync + Send,
N5: N5Writer + Sync + Send + Clone + 'static,
SliceDataBlock<T, std::sync::Arc<[T]>>: n5::WriteableDataBlock {

fn write<T, N5>(n: &N5, compression: &CompressionType, block_data: &[T], pool_size: usize)
where
T: 'static + std::fmt::Debug + ReflectedType + PartialEq + Default + Sync + Send,
N5: N5Writer + Sync + Send + Clone + 'static,
SliceDataBlock<T, std::sync::Arc<[T]>>: n5::WriteableDataBlock,
{
let block_size = smallvec![BLOCK_DIM; 3];
let data_attrs = DatasetAttributes::new(
smallvec![u64::from(BLOCK_DIM) * N_BLOCKS; 3],
Expand All @@ -83,9 +77,11 @@ fn write<T, N5>(
compression.clone(),
);

let path_name = format!("dataset.{:?}.{}",
let path_name = format!(
"dataset.{:?}.{}",
data_attrs.get_data_type(),
data_attrs.get_compression());
data_attrs.get_compression()
);

n.create_dataset(&path_name, &data_attrs)
.expect("Failed to create dataset");
Expand All @@ -104,10 +100,7 @@ fn write<T, N5>(
let pn = path_name.clone();
let da = data_attrs.clone();
all_jobs.push(pool.spawn_fn(move || {
let block_in = SliceDataBlock::new(
bs,
smallvec![x, y, z],
bd);
let block_in = SliceDataBlock::new(bs, smallvec![x, y, z], bd);
ni.write_block(&pn, &da, &block_in)
.expect("Failed to write block");
Ok(0)
Expand All @@ -120,30 +113,36 @@ fn write<T, N5>(
}

fn bench_write_dtype_compression<T, C>(b: &mut Bencher, pool_size: usize)
where
T: 'static + ReflectedType + Default + PartialEq + std::fmt::Debug +
std::convert::From<i8> + Sync + Send,
C: compression::Compression,
CompressionType: std::convert::From<C>,
SliceDataBlock<T, std::sync::Arc<[T]>>: n5::WriteableDataBlock {

where
T: 'static
+ ReflectedType
+ Default
+ PartialEq
+ std::fmt::Debug
+ std::convert::From<i8>
+ Sync
+ Send,
C: compression::Compression,
CompressionType: std::convert::From<C>,
SliceDataBlock<T, std::sync::Arc<[T]>>: n5::WriteableDataBlock,
{
let dir = tempdir::TempDir::new("rust_n5_integration_tests").unwrap();

let n = N5Filesystem::open_or_create(dir.path())
.expect("Failed to create N5 filesystem");
let n = N5Filesystem::open_or_create(dir.path()).expect("Failed to create N5 filesystem");
let compression = CompressionType::new::<C>();
// TODO: load the test image data.
// let block_data: Vec<T> = vec![T::default(); (BLOCK_DIM * BLOCK_DIM * BLOCK_DIM) as usize];
let block_data = TEST_IMAGE.iter().take((BLOCK_DIM * BLOCK_DIM * BLOCK_DIM) as usize)
let block_data = TEST_IMAGE
.iter()
.take((BLOCK_DIM * BLOCK_DIM * BLOCK_DIM) as usize)
.map(|&v| T::from(v))
.collect::<Vec<T>>();

b.iter(|| write(&n, &compression, &block_data, pool_size));

b.bytes =
(BLOCK_DIM * BLOCK_DIM * BLOCK_DIM) as u64 *
(N_BLOCKS * N_BLOCKS * N_BLOCKS) as u64 *
std::mem::size_of::<T>() as u64;
b.bytes = (BLOCK_DIM * BLOCK_DIM * BLOCK_DIM) as u64
* (N_BLOCKS * N_BLOCKS * N_BLOCKS) as u64
* std::mem::size_of::<T>() as u64;
}

// 1 Thread. Can't macro this because of the concat_idents! limitation.
Expand Down
29 changes: 12 additions & 17 deletions benches/simple.rs
Original file line number Diff line number Diff line change
@@ -1,33 +1,28 @@
//! # Simple In-memory Read/Write Benchmarks
#![feature(test)]


extern crate test;


use rand::{
distributions::Standard,
Rng,
};
use test::Bencher;

use n5::prelude::*;
use n5::smallvec::smallvec;
use n5::{
DefaultBlock,
DefaultBlockReader,
DefaultBlockWriter,
};
use n5::smallvec::smallvec;


fn test_block_compression_rw<T>(
compression: compression::CompressionType,
b: &mut Bencher
) where T: 'static + std::fmt::Debug + ReflectedType + PartialEq + Default,
rand::distributions::Standard: rand::distributions::Distribution<T>,
VecDataBlock<T>: n5::ReadableDataBlock + n5::WriteableDataBlock,
fn test_block_compression_rw<T>(compression: compression::CompressionType, b: &mut Bencher)
where
T: 'static + std::fmt::Debug + ReflectedType + PartialEq + Default,
rand::distributions::Standard: rand::distributions::Distribution<T>,
VecDataBlock<T>: n5::ReadableDataBlock + n5::WriteableDataBlock,
{

let data_attrs = DatasetAttributes::new(
smallvec![1024, 1024, 1024],
smallvec![64, 64, 64],
Expand All @@ -41,20 +36,20 @@ fn test_block_compression_rw<T>(
let block_in = VecDataBlock::new(
data_attrs.get_block_size().into(),
smallvec![0, 0, 0],
block_data.clone());
block_data.clone(),
);

let mut inner: Vec<u8> = Vec::new();

b.iter(|| {
DefaultBlock::write_block(
&mut inner,
&data_attrs,
&block_in).expect("write_block failed");
DefaultBlock::write_block(&mut inner, &data_attrs, &block_in).expect("write_block failed");

let _block_out = <DefaultBlock as DefaultBlockReader<T, _>>::read_block(
&inner[..],
&data_attrs,
smallvec![0, 0, 0]).expect("read_block failed");
smallvec![0, 0, 0],
)
.expect("read_block failed");
});

b.bytes = (data_attrs.get_block_num_elements() * data_attrs.get_data_type().size_of()) as u64;
Expand Down
11 changes: 11 additions & 0 deletions release.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
publish = false
pre-release-commit-message = "Version {{version}}"
pre-release-replacements = [
{file="CHANGELOG.md", search="^## \\[Unreleased\\]", replace="## [{{version}}] - {{date}}", exactly=1},
{file="CHANGELOG.md", search="\\[Unreleased\\]", replace="[{{version}}]", min=1},
{file="CHANGELOG.md", search="\\.\\.\\.HEAD", replace="...{{tag_name}}", exactly=1},
]
post-release-replacements = [
{file="CHANGELOG.md", search="<!-- next-header -->", replace="<!-- next-header -->\n## [Unreleased]\n\n", exactly=1},
{file="CHANGELOG.md", search="<!-- next-url -->", replace="<!-- next-url -->\n[Unreleased]: https://github.com/aschampion/rust-n5/compare/{{tag_name}}...HEAD", exactly=1},
]
Loading

0 comments on commit a5664a6

Please sign in to comment.