random cleanups and fix clippy (#25)

* random cleanups

* fix clippy

* fix gitlab yaml

* fmt

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: check
-          args: --workspace
+          args: --workspace --all-features
 
   test:
     name: Test
@@ -85,18 +85,17 @@ jobs:
           command: fmt
           args: --all -- --check
 
-  # clippy:
-  #   name: Clippy
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: actions-rs/toolchain@v1
-  #       with:
-  #         profile: minimal
-  #         toolchain: stable
-  #         override: true
-  #     - ru
-  #     - uses: actions-rs/cargo@v1
-  #       with:
-  #         command: clippy
-  #         args: --workspace
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --workspace --all-features

.gitignore

@@ -1,2 +1,5 @@
 /target
 .DS_Store
+.vscode
+**/hfuzz_target
+**/hfuzz_workspace

README.md

@@ -1,16 +1,31 @@
 # reed-solomon-novelpoly
 
-An implementation of  Novel Polynomial Basis and its Application to Reed-Solomon Erasure Codes [1] [2] .
+An implementation of  Novel Polynomial Basis and its Application to Reed-Solomon Erasure Codes [1] [2] [3].
 
 Runs encoding and reconstruction in `O(n lg(n))`. Note that for small number `n` there is a static offset due to a walsh transform over the full domain in reconstruction.
 
 ## Goals
 
 Be really fast for `n > 100`.
 
-## Non-goals
+## Benchmarking
 
-TODO
+For benchmarking the implementation against itself and the naive implementation, `criterion` is used.
+
+```sh
+cargo bench
+```
+
+## Fuzzing
+
+Currently `honggfuzz` is used.
+
+Install `cargo install cargo-hongg` and run with:
+
+```sh
+cargo-hongg fuzz --bin <binary_name>
+```
 
 [1]: https://www.citi.sinica.edu.tw/papers/whc/4454-F.pdf
 [2]: https://arxiv.org/abs/1404.3458
+[3]: https://www.citi.sinica.edu.tw/papers/whc/5524-F.pdf

reed-solomon-benches/Cargo.toml

@@ -31,3 +31,4 @@ default = []
 novelpoly-cxx = ["reed-solomon-novelpoly/with-alt-cxx-impl"]
 naive = ["reed-solomon-erasure"]
 upperbounds = ["naive"]
+avx = ["reed-solomon-novelpoly/avx"]

reed-solomon-benches/benches/criterion.rs

@@ -233,6 +233,7 @@ pub mod parameterized {
 	) {
 		use reed_solomon_novelpoly::f2e16::Additive;
 		use rand::Rng;
+		#[cfg(feature = "avx")]
 		{
 			group.bench_with_input(
 				BenchmarkId::new("novel-poly-guts-encode-faster8", param.to_string()),
@@ -267,6 +268,7 @@ pub mod parameterized {
 				})
 			});
 		}
+		#[cfg(feature = "avx")]
 		{
 			group.bench_with_input(
 				BenchmarkId::new("novel-poly-encode-sub-faster8", param.to_string()),
@@ -285,15 +287,15 @@ pub mod parameterized {
 				let dist = rand::distributions::Uniform::new_inclusive(u8::MIN, u8::MAX);
 				let data = Vec::from_iter(rng.sample_iter::<u8, _>(dist).take(k * 2));
 				b.iter(|| {
-					reed_solomon_novelpoly::f2e16::encode_sub_plain(black_box(&data), black_box(n), black_box(k));
+					reed_solomon_novelpoly::f2e16::encode_sub_plain(black_box(&data), black_box(n), black_box(k))
+						.unwrap();
 				})
 			});
 		}
 	}
 
 	pub fn bench_encode_guts(crit: &mut Criterion) {
 		let mut rng = SmallRng::from_seed(SMALL_RNG_SEED);
-		use reed_solomon_novelpoly::f2e16::{Additive8x};
 
 		// factors of 1/2..1/8 are reasonable
 		for f_exp in 1..3 {
@@ -303,8 +305,12 @@ pub mod parameterized {
 				let k = 1 << k_exp;
 				let n = k * f;
 				assert!(n > k);
-				assert_eq!(n % Additive8x::LANE, 0);
-				assert_eq!(k % Additive8x::LANE, 0);
+				#[cfg(feature = "avx")]
+				{
+					use reed_solomon_novelpoly::f2e16::{Additive8x};
+					assert_eq!(n % Additive8x::LANE, 0);
+					assert_eq!(k % Additive8x::LANE, 0);
+				}
 				let param = format!("n={n} k={k} (n/k = {f})");
 				encode_guts_add_to_group(&mut group, param, n, k, &mut rng);
 			}
@@ -318,14 +324,14 @@ fn parameterized_criterion() -> Criterion {
 }
 
 criterion_group!(
-name = plot_parameterized;
-config = parameterized_criterion();
-targets =
-	parameterized::bench_encode_2d,
-	parameterized::bench_reconstruct_2d,
-	parameterized::bench_encode_fixed_1mb_payload,
-	parameterized::bench_reconstruct_fixed_1mb_payload,
-	parameterized::bench_encode_guts,
+	name = plot_parameterized;
+	config = parameterized_criterion();
+	targets =
+		parameterized::bench_encode_2d,
+		parameterized::bench_reconstruct_2d,
+		parameterized::bench_encode_fixed_1mb_payload,
+		parameterized::bench_reconstruct_fixed_1mb_payload,
+		parameterized::bench_encode_guts,
 );
 
 #[cfg(feature = "upperbounds")]

reed-solomon-benches/src/naive/mod.rs

@@ -49,7 +49,7 @@ pub fn reconstruct<S: Shard>(
 	// Try to reconstruct missing shards
 	r.reconstruct_data(&mut received_shards).expect("Sufficient shards must be received. qed");
 
-	let result = received_shards.into_iter().filter_map(|x| x).take(r.data_shard_count()).fold(
+	let result = received_shards.into_iter().flatten().take(r.data_shard_count()).fold(
 		Vec::with_capacity(12 << 20),
 		|mut acc, reconstructed_shard| {
 			acc.extend_from_slice(AsRef::<[u8]>::as_ref(&reconstructed_shard));

reed-solomon-novelpoly/Cargo.toml

@@ -19,9 +19,6 @@ bindgen = { version = "0.66.1", optional = true }
 cc = { version = "1.0.67", features = ["parallel"], optional = true }
 
 [dependencies]
-reed-solomon-erasure = { version = "6.0", features = [
-  "simd-accel",
-], optional = true }
 static_init = "1"
 
 thiserror = "1.0.23"
@@ -42,7 +39,7 @@ assert_matches = "1.5.0"
 [features]
 default = []
 with-alt-cxx-impl = ["cc", "bindgen"]
-naive = ["reed-solomon-erasure"]
 mock = ["rand", "reed-solomon-tester"]
 # enable the small field, experimental support
 f256 = []
+avx = []

reed-solomon-novelpoly/src/cxx.rs

@@ -3,6 +3,7 @@
 #![allow(non_snake_case)]
 #![allow(dead_code)]
 
+#[allow(clippy::module_inception)]
 mod cxx {
 	include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
 }

reed-solomon-novelpoly/src/field/inc_encode.rs

@@ -202,7 +202,7 @@ pub fn encode_sub_plain(bytes: &[u8], n: usize, k: usize) -> Result<Vec<Additive
 	let zero_bytes_to_add = n * 2 - bytes_len;
 	let mut elm_data = Vec::with_capacity(n);
 	let zeros = std::iter::repeat(&0u8).take(zero_bytes_to_add);
-	for (first, second) in bytes.into_iter().chain(zeros).tuples() {
+	for (first, second) in bytes.iter().chain(zeros).tuples() {
 		elm_data.push(Additive(Elt::from_be_bytes([*first, *second])));
 	}
 

reed-solomon-novelpoly/src/field/inc_log_mul.rs

@@ -38,6 +38,7 @@ impl Additive {
 
 	/// Return a*EXP_TABLE[b] over GF(2^r)
     #[inline(always)]
+	#[allow(clippy::should_implement_trait)]
 	pub fn mul(self, other: Multiplier) -> Additive {
 		if self == Self::ZERO {
 			return Self::ZERO;

reed-solomon-novelpoly/src/lib.rs

@@ -1,4 +1,5 @@
 #![forbid(unused_crate_dependencies)]
+#![allow(clippy::needless_range_loop)]
 
 pub mod errors;
 pub use errors::*;
@@ -33,12 +34,6 @@ mod test {
 	use super::*;
 	use reed_solomon_tester::{roundtrip, BYTES, N_SHARDS};
 
-	#[cfg(feature = "naive")]
-	#[test]
-	fn status_quo_roundtrip() -> Result<()> {
-		roundtrip(status_quo::encode::<WrappedShard>, status_quo::reconstruct::<WrappedShard>, &BYTES[..1337], N_SHARDS)
-	}
-
 	#[test]
 	fn novel_poly_basis_roundtrip() -> Result<()> {
 		roundtrip(

reed-solomon-novelpoly/src/novel_poly_basis/mod.rs

@@ -130,11 +130,14 @@ impl ReedSolomon {
 		let k2 = self.k * 2;
 		// prepare one wrapped shard per validator
 		let mut shards = vec![
-			<S as From<Vec<u8>>>::from({
-				let mut v = Vec::<u8>::with_capacity(shard_len);
-				unsafe { v.set_len(shard_len) }
-				v
-			});
+			<S as From<Vec<u8>>>::from(
+				#[allow(clippy::uninit_vec)]
+				{
+					let mut v = Vec::<u8>::with_capacity(shard_len);
+					unsafe { v.set_len(shard_len) }
+					v
+				}
+			);
 			validator_count
 		];
 

reed-solomon-novelpoly/src/novel_poly_basis/reconstruct.rs

@@ -1,7 +1,7 @@
 use super::*;
 
 /// each shard contains one symbol of one run of erasure coding
-pub fn reconstruct<'a, S: Shard>(received_shards: Vec<Option<S>>, validator_count: usize) -> Result<Vec<u8>> {
+pub fn reconstruct<S: Shard>(received_shards: Vec<Option<S>>, validator_count: usize) -> Result<Vec<u8>> {
 	let params = CodeParams::derive_parameters(validator_count, recoverablity_subset_size(validator_count))?;
 
 	let rs = params.make_encoder();