Version 0.5.0

README.md

@@ -22,10 +22,8 @@ An implementation in Rust is planned.
 
 Raw files are the foundation of data stored on S5. They can have any size and are stored as the hash of themselves (blake3 by default).
 No additional metadata like filename or content type is added, to make deduplication as efficient as possible.
-
-### Metadata files/CIDs
-
-Metadata files are stored as raw files, but are usually very small. They reference one or more raw files that contain the file content and have additional metadata like filename, content type and hashes for specific chunks of the referenced raw file to make trustless streaming possible.
+Raw files are stored together with a part of their [BLAKE3 Bao](https://github.com/oconnor663/bao) hash tree, which makes it possible
+to trustlessly stream any 256 KiB chunk of a file. The chunk size is configurable.
 
 ### Directory files/CIDs
 
@@ -36,14 +34,13 @@ Can be used to deploy web apps on S5.
 ### Registry
 
 The registry is a decentralized key-value store with realtime subscriptions.
-Keys consist of a public ed25519 key and a 32-byte arbitrary datakey.
-The values have a revision number and are limited to 48 bytes in size.
-All writes to an entry must be signed by the public key that's part of the key.
+Keys are public ed25519 keys.
+The values have a revision number and are limited to 48 bytes of data.
+All writes to an entry must be signed by the public key.
 
 ### Resolver CIDs
 
 Resolver cids are just registry entries encoded as a CID.
-Registry entries used in resolver CIDs must have an empty datakey of just zeros.
 They reference another CID and are for example used for dynamically updating websites.
 
 ## Comparison to IPFS
@@ -64,7 +61,7 @@ S5 currently supports three storage backends:
 - S3 (Any cloud provider supporting the S3 protocol, see https://s3.wiki)
 - Local filesystem (needs additional configuration to make a http port available on the internet)
 - Arweave (expensive, permanent storage)
-- Sia (https://sia.tech/, planned)
+- Sia (experimental, https://sia.tech/)
 
 S5 currently supports one protocol to establish a connection between nodes:
 - TCP

bin/ffi.io.dart

@@ -0,0 +1,12 @@
+import 'dart:ffi';
+import 'dart:io';
+
+import 'package:s5_server/rust/bridge_generated.dart';
+
+RustImpl initializeExternalLibrary(String path) {
+  return RustImpl(
+    Platform.isMacOS || Platform.isIOS
+        ? DynamicLibrary.executable()
+        : DynamicLibrary.open(path),
+  );
+}

bin/s5_server.dart

@@ -1,14 +1,19 @@
+import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
+import 'dart:math';
 import 'dart:typed_data';
 
-import 'package:base_codecs/base_codecs.dart';
-import 'package:cryptography/helpers.dart';
-import 'package:s5_server/constants.dart';
+import 'package:lib5/lib5.dart';
 import 'package:tint/tint.dart';
 import 'package:toml/toml.dart';
+import 'package:cryptography/cryptography.dart';
 
+import 'package:s5_server/constants.dart';
 import 'package:s5_server/node.dart';
 import 'package:s5_server/logger/console.dart';
+import 'package:s5_server/rust/bridge_definitions.dart';
+import 'ffi.io.dart';
 
 void main(List<String> arguments) async {
   final logger = ConsoleLogger();
@@ -27,17 +32,19 @@ void main(List<String> arguments) async {
   logger.info('s5-dart'.green().bold() + ' ' + 'v$nodeVersion'.red().bold());
   logger.info('');
 
+  final rust = initializeExternalLibrary('rust/target/release/librust.so');
+  final crypto = RustCryptoImplementation(rust);
+
   if (file
       .readAsStringSync()
       .contains('"AUTOMATICALLY_GENERATED_ON_FIRST_START"')) {
     logger.info('Generating seed...');
-    final seed = Uint8List(64);
-    fillBytesWithSecureRandom(seed);
+    final seed = crypto.generateRandomBytes(32);
 
     file.writeAsStringSync(
       file.readAsStringSync().replaceFirst(
             '"AUTOMATICALLY_GENERATED_ON_FIRST_START"',
-            '"${base58Bitcoin.encode(seed)}"',
+            '"${base64Url.encode(seed)}"',
           ),
     );
   }
@@ -46,8 +53,82 @@ void main(List<String> arguments) async {
 
   final node = S5Node(
     config,
-    logger,
+    logger: logger,
+    rust: rust,
+    crypto: crypto,
   );
 
-  await node.start();
+  runZonedGuarded(
+    node.start,
+    (e, st) {
+      logger.catched(e, st);
+    },
+  );
+}
+
+class RustCryptoImplementation extends CryptoImplementation {
+  final Rust rust;
+
+  RustCryptoImplementation(this.rust);
+
+  final ed25519 = Ed25519();
+  final _defaultSecureRandom = Random.secure();
+
+  @override
+  Uint8List generateRandomBytes(int length) {
+    final bytes = Uint8List(length);
+
+    for (var i = 0; i < bytes.length; i++) {
+      bytes[i] = _defaultSecureRandom.nextInt(256);
+    }
+
+    return bytes;
+  }
+
+  @override
+  Future<Uint8List> hashBlake3(Uint8List input) {
+    return rust.hashBlake3(input: input);
+  }
+
+  @override
+  Future<KeyPairEd25519> newKeyPairEd25519({required Uint8List seed}) async {
+    final keyPair = await ed25519.newKeyPairFromSeed(seed);
+    final pk = (await keyPair.extractPublicKey()).bytes;
+    return KeyPairEd25519(Uint8List.fromList(seed + pk));
+  }
+
+  @override
+  Future<Uint8List> signEd25519({
+    required KeyPairEd25519 kp,
+    required Uint8List message,
+  }) async {
+    final signature = await ed25519.sign(
+      message,
+      keyPair: SimpleKeyPairData(kp.extractBytes().sublist(0, 32),
+          publicKey: SimplePublicKey(
+            kp.extractBytes().sublist(32),
+            type: KeyPairType.ed25519,
+          ),
+          type: KeyPairType.ed25519),
+    );
+    return Uint8List.fromList(signature.bytes);
+  }
+
+  @override
+  Future<bool> verifyEd25519({
+    required Uint8List pk,
+    required Uint8List message,
+    required Uint8List signature,
+  }) async {
+    return ed25519.verify(
+      message,
+      signature: Signature(
+        signature,
+        publicKey: SimplePublicKey(
+          pk,
+          type: KeyPairType.ed25519,
+        ),
+      ),
+    );
+  }
 }

default_config.toml

@@ -15,9 +15,9 @@ port = 5522
 [http.api.delete]
 enabled = false
 
-[network.peers]
+[p2p.peers]
 initial = [
   'tcp://[email protected]:43221', # uber.space
   'tcp://[email protected]:4444', # hetzner
-  'tcp://z2DVYoNRUFknwbFPz74j6sz8dZ1LosYzQ8XjjSCNcGHajcC@168.119.116.125:4444', # test node
+  'tcp://z2DVNifJx5XhjwQpecQMCZCguum2sAnJNNLfrQmRZt2zxiq@168.119.116.125:4444', # test node
 ]

lib/accounts/user.dart

@@ -0,0 +1,12 @@
+class User {
+  final int id;
+  final int createdAt;
+  final String? email;
+  int get tier => 1;
+
+  User({
+    required this.id,
+    required this.createdAt,
+    required this.email,
+  });
+}

lib/constants.dart

@@ -1,39 +1,8 @@
-const nodeVersion = '0.3.0';
+// ! S5 node version
+const nodeVersion = '0.5.0';
 
-const defaultChunkSize = 1024 * 1024;
+// ! default chunk size for hashes
+const defaultChunkSize = 256 * 1024;
 
-// These bytes are carefully selected to make the base58 and base32 representations of different CID types
-// easy to distinguish and not collide with anything on https://github.com/multiformats/multicodec
-const cidTypeRaw = 0x26;
-const cidTypeMetadataFile = 0x2d;
-const cidTypeMetadataDirectory = 0x59;
-const cidTypeResolver = 0x25;
+// const magicByteStoredFile = 0x8d;
 
-const registryS5MagicByte = 0x5a;
-const metadataMagicByte = 0x5f;
-
-// types for metadata files
-const metadataTypeFile = 0x01;
-const metadataTypeChunkedFile = 0x02;
-const metadataTypeDirectory = 0x03;
-
-const registryMaxDataSize = 48;
-
-// const mhashSha256 = [0x12, 0x20];
-const mhashBlake3 = [0x1e, 0x20];
-
-const mkeyEd25519 = 0xed;
-
-//  Use this for protocol updates
-const protocolMethodHandshakeOpen = 1;
-const protocolMethodHandshakeDone = 2;
-
-const protocolMethodSignedMessage = 10;
-
-const protocolMethodHashQueryResponse = 5;
-const protocolMethodHashQuery = 4;
-
-const protocolMethodAnnouncePeers = 7;
-
-const protocolMethodRegistryUpdate = 12;
-const protocolMethodRegistryQuery = 13;