make leafmap work

utreexo · Sep 3, 2024 · da0dcb7 · da0dcb7
1 parent 5b7c2cf
commit da0dcb7
Show file tree

Hide file tree

Showing 7 changed files with 276 additions and 137 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -6,8 +6,11 @@ SUBDIRS = include
 #libutreexo_cpp_la_SOURCES = include/cpp/utreexo.cpp
 #libutreexo_cpp_la_LDFLAGS = -version-info 0:1:0
 
-check_PROGRAMS = test_flat_file test_forest
-test_flat_file_SOURCES = src/test_flat_file.c 
+check_PROGRAMS = test_flat_file test_forest test_leaf_map
+
+test_flat_file_SOURCES = src/test_flat_file.c
+
+test_leaf_map_SOURCES = src/test_leaf_map.c
 
 test_forest_SOURCES = src/forest_test.c
 test_forest_LDADD = libutreexo.la -lcrypto

diff --git a/src/forest_test.c b/src/forest_test.c
@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stddef.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -21,7 +22,7 @@ static inline struct utreexo_forest get_test_forest(const char *filename) {
   sprintf(map_name, "map_%s", filename);
 
   utreexo_leaf_map map;
-  utreexo_leaf_map_new(&map, map_name, O_CREAT | O_RDWR);
+  utreexo_leaf_map_new(&map, map_name, O_CREAT | O_RDWR, NULL);
 
   struct utreexo_forest p = {
       .data = file,
@@ -160,7 +161,7 @@ void test_add_single() {
   utreexo_forest_file_init(&file, &heap, "add_single.bin");
 
   utreexo_leaf_map leaf_map;
-  utreexo_leaf_map_new(&leaf_map, "leaves_single.bin", O_CREAT | O_RDWR);
+  utreexo_leaf_map_new(&leaf_map, "leaves_single.bin", O_CREAT | O_RDWR, NULL);
 
   struct utreexo_forest p = {
       .data = file,
@@ -230,6 +231,7 @@ void test_delete_with_map() {
 
   utreexo_forest_node *pnode = NULL;
   utreexo_leaf_map_get(&p.leaf_map, &pnode, leaf);
+  assert(pnode != NULL);
   delete_single(&p, pnode);
 
   ASSERT_ARRAY_EQ(expected_root, p.roots[3]->hash.hash, 32);
@@ -315,103 +317,103 @@ void test_delete_some() {
   TEST_END;
 }
 
-static deletion_test_data test_cases[] =
-    {{
-         .expected_roots = {{0x33, 0x2c, 0x30, 0x61, 0x88, 0xd3, 0x5e, 0xb2,
-                             0x2e, 0xcb, 0x05, 0xd8, 0xc0, 0x04, 0x46, 0xcd,
-                             0x6a, 0x7a, 0x47, 0x5f, 0x66, 0x15, 0xf4, 0x62,
-                             0x07, 0xcf, 0xaa, 0x71, 0x3b, 0xb3, 0xe6, 0x2c}},
-         .expected_roots_len = 1,
-         .leaf_preimages = (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7},
-         .preimage_count = 8,
-         .target_values = (uint64_t[]){1, 7},
-         .n_target_values = 2,
-     },
-     {
-         .expected_roots = {{0x3b, 0x8b, 0x6e, 0xb2, 0x31, 0x43, 0x70, 0x92,
-                             0xf6, 0xd9, 0xfb, 0xf8, 0xa0, 0x69, 0x6b, 0x7c,
-                             0xb4, 0x46, 0xe4, 0x0f, 0x1b, 0xf8, 0x1d, 0xdb,
-                             0x23, 0xd3, 0xea, 0xbb, 0x30, 0x80, 0xb0, 0xdd}},
-         .expected_roots_len = 1,
-         .leaf_preimages = (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7},
-         .preimage_count = 8,
-         .target_values = (uint64_t[]){1, 5, 7},
-         .n_target_values = 3,
-     },
-     {
-         .expected_roots = {{0x37, 0x96, 0x8e, 0xf7, 0x3d, 0x30, 0xdd, 0xa3,
-                             0x8e, 0xde, 0x83, 0x57, 0xd6, 0x65, 0x93, 0xc7,
-                             0x2a, 0xcd, 0x4f, 0x0e, 0xb9, 0xf7, 0xa1, 0xa9,
-                             0xac, 0xfe, 0xb7, 0xde, 0x85, 0x0c, 0x05, 0xb4},
-                            {0x3c, 0xb9, 0x20, 0xc1, 0x13, 0xe8, 0xce, 0x3a,
-                             0xce, 0x35, 0xfc, 0x83, 0x5e, 0x6f, 0xf8, 0x3a,
-                             0x65, 0x66, 0xc4, 0xc1, 0x27, 0xea, 0x67, 0xee,
-                             0xbc, 0x0f, 0x5d, 0x25, 0xd2, 0x29, 0x5e, 0xa2}},
-         .expected_roots_len = 2,
-         .leaf_preimages = (uint64_t[]){0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
-                                        10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
-         .preimage_count = 20,
-         .target_values = (uint64_t[]){1, 10},
-         .n_target_values = 2,
-     },
-     {
-         .expected_roots = {{0x37, 0x96, 0x8e, 0xf7, 0x3d, 0x30, 0xdd, 0xa3,
-                             0x8e, 0xde, 0x83, 0x57, 0xd6, 0x65, 0x93, 0xc7,
-                             0x2a, 0xcd, 0x4f, 0x0e, 0xb9, 0xf7, 0xa1, 0xa9,
-                             0xac, 0xfe, 0xb7, 0xde, 0x85, 0x0c, 0x05, 0xb4},
-                            {0x21, 0x32, 0x6d, 0x8a, 0xeb, 0xeb, 0x6e, 0xf7,
-                             0xbc, 0x02, 0xf4, 0x0b, 0xdf, 0x77, 0x8a, 0x02,
-                             0xba, 0x1c, 0x83, 0x6b, 0x25, 0x7f, 0x94, 0x6a,
-                             0xe2, 0x1c, 0xab, 0x2a, 0x6f, 0x95, 0xfa, 0x18}},
-         .expected_roots_len = 2,
-         .leaf_preimages = (uint64_t[]){0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
-                                        10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
-         .preimage_count = 20,
-         .target_values = (uint64_t[]){1, 10, 16},
-         .n_target_values = 3,
-     },
-     {
-         .expected_roots =
-             {{0xe4, 0x67, 0x6b, 0xa6, 0x3c, 0x94, 0x58, 0x8b, 0xcb, 0x40, 0x83,
-               0xe5, 0x47, 0x40, 0x63, 0xb6, 0x33, 0xaa, 0x63, 0x18, 0x97, 0xdd,
-               0xaa, 0xbd, 0xe7, 0xa4, 0xda, 0x45, 0x93, 0x68, 0x90, 0xcb},
-              {0xca, 0xe9, 0x21, 0xbb, 0xf6, 0x49, 0xd4, 0xdd, 0x84, 0xc2, 0x52,
-               0xc5, 0x5c, 0x54, 0x0e, 0x2e, 0x30, 0xc6, 0xc0, 0x0c, 0xb0, 0x89,
-               0xd9, 0x70, 0x4b, 0xd6, 0x13, 0xec, 0xea, 0x30, 0x86, 0x43},
-              {0x55, 0xd0, 0xa0, 0xef, 0x8f, 0x5c, 0x25, 0xa9, 0xda, 0x26, 0x6b,
-               0x36, 0xc0, 0xc5, 0xf4, 0xb3, 0x10, 0x08, 0xec, 0xe8, 0x2d, 0xf2,
-               0x51, 0x2c, 0x89, 0x66, 0xbd, 0xdc, 0xc2, 0x7a, 0x66, 0xa0},
-              {0x4d, 0x7b, 0x3e, 0xf7, 0x30, 0x0a, 0xcf, 0x70, 0xc8, 0x92, 0xd8,
-               0x32, 0x7d, 0xb8, 0x27, 0x2f, 0x54, 0x43, 0x4a, 0xdb, 0xc6, 0x1a,
-               0x4e, 0x13, 0x0a, 0x56, 0x3c, 0xb5, 0x9a, 0x0d, 0x0f, 0x47}},
-         .expected_roots_len = 4,
-         .leaf_preimages =
-             (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
-         .preimage_count = 15,
-         .target_values = (uint64_t[]){1, 10, 5},
-         .n_target_values = 3,
-     },
-     {
-         .expected_roots =
-             {{0x35, 0x2d, 0x0d, 0x5e, 0x17, 0x23, 0x16, 0xe7, 0x3b, 0x6d, 0x0d,
-               0x40, 0x60, 0x5a, 0xd4, 0x11, 0xc7, 0x9b, 0x7e, 0x7c, 0x1e, 0xd0,
-               0xb4, 0x52, 0x9c, 0x56, 0x5f, 0x08, 0x05, 0x7b, 0xc4, 0xa8},
-              {0xca, 0xe9, 0x21, 0xbb, 0xf6, 0x49, 0xd4, 0xdd, 0x84, 0xc2, 0x52,
-               0xc5, 0x5c, 0x54, 0x0e, 0x2e, 0x30, 0xc6, 0xc0, 0x0c, 0xb0, 0x89,
-               0xd9, 0x70, 0x4b, 0xd6, 0x13, 0xec, 0xea, 0x30, 0x86, 0x43},
-              {0x55, 0xd0, 0xa0, 0xef, 0x8f, 0x5c, 0x25, 0xa9, 0xda, 0x26, 0x6b,
-               0x36, 0xc0, 0xc5, 0xf4, 0xb3, 0x10, 0x08, 0xec, 0xe8, 0x2d, 0xf2,
-               0x51, 0x2c, 0x89, 0x66, 0xbd, 0xdc, 0xc2, 0x7a, 0x66, 0xa0},
-              {0x4d, 0x7b, 0x3e, 0xf7, 0x30, 0x0a, 0xcf, 0x70, 0xc8, 0x92, 0xd8,
-               0x32, 0x7d, 0xb8, 0x27, 0x2f, 0x54, 0x43, 0x4a, 0xdb, 0xc6, 0x1a,
-               0x4e, 0x13, 0x0a, 0x56, 0x3c, 0xb5, 0x9a, 0x0d, 0x0f, 0x47}},
-         .expected_roots_len = 4,
-         .leaf_preimages =
-             (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
-         .preimage_count = 15,
-         .target_values = (uint64_t[]){10, 5, 16},
-         .n_target_values = 3,
-     }};
+static deletion_test_data test_cases[] = {
+    {
+        .expected_roots = {{0x33, 0x2c, 0x30, 0x61, 0x88, 0xd3, 0x5e, 0xb2,
+                            0x2e, 0xcb, 0x05, 0xd8, 0xc0, 0x04, 0x46, 0xcd,
+                            0x6a, 0x7a, 0x47, 0x5f, 0x66, 0x15, 0xf4, 0x62,
+                            0x07, 0xcf, 0xaa, 0x71, 0x3b, 0xb3, 0xe6, 0x2c}},
+        .expected_roots_len = 1,
+        .leaf_preimages = (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7},
+        .preimage_count = 8,
+        .target_values = (uint64_t[]){1, 7},
+        .n_target_values = 2,
+    },
+    {
+        .expected_roots = {{0x3b, 0x8b, 0x6e, 0xb2, 0x31, 0x43, 0x70, 0x92,
+                            0xf6, 0xd9, 0xfb, 0xf8, 0xa0, 0x69, 0x6b, 0x7c,
+                            0xb4, 0x46, 0xe4, 0x0f, 0x1b, 0xf8, 0x1d, 0xdb,
+                            0x23, 0xd3, 0xea, 0xbb, 0x30, 0x80, 0xb0, 0xdd}},
+        .expected_roots_len = 1,
+        .leaf_preimages = (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7},
+        .preimage_count = 8,
+        .target_values = (uint64_t[]){1, 5, 7},
+        .n_target_values = 3,
+    },
+    {
+        .expected_roots = {{0x37, 0x96, 0x8e, 0xf7, 0x3d, 0x30, 0xdd, 0xa3,
+                            0x8e, 0xde, 0x83, 0x57, 0xd6, 0x65, 0x93, 0xc7,
+                            0x2a, 0xcd, 0x4f, 0x0e, 0xb9, 0xf7, 0xa1, 0xa9,
+                            0xac, 0xfe, 0xb7, 0xde, 0x85, 0x0c, 0x05, 0xb4},
+                           {0x3c, 0xb9, 0x20, 0xc1, 0x13, 0xe8, 0xce, 0x3a,
+                            0xce, 0x35, 0xfc, 0x83, 0x5e, 0x6f, 0xf8, 0x3a,
+                            0x65, 0x66, 0xc4, 0xc1, 0x27, 0xea, 0x67, 0xee,
+                            0xbc, 0x0f, 0x5d, 0x25, 0xd2, 0x29, 0x5e, 0xa2}},
+        .expected_roots_len = 2,
+        .leaf_preimages = (uint64_t[]){0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+                                       10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
+        .preimage_count = 20,
+        .target_values = (uint64_t[]){1, 10},
+        .n_target_values = 2,
+    },
+    {
+        .expected_roots = {{0x37, 0x96, 0x8e, 0xf7, 0x3d, 0x30, 0xdd, 0xa3,
+                            0x8e, 0xde, 0x83, 0x57, 0xd6, 0x65, 0x93, 0xc7,
+                            0x2a, 0xcd, 0x4f, 0x0e, 0xb9, 0xf7, 0xa1, 0xa9,
+                            0xac, 0xfe, 0xb7, 0xde, 0x85, 0x0c, 0x05, 0xb4},
+                           {0x21, 0x32, 0x6d, 0x8a, 0xeb, 0xeb, 0x6e, 0xf7,
+                            0xbc, 0x02, 0xf4, 0x0b, 0xdf, 0x77, 0x8a, 0x02,
+                            0xba, 0x1c, 0x83, 0x6b, 0x25, 0x7f, 0x94, 0x6a,
+                            0xe2, 0x1c, 0xab, 0x2a, 0x6f, 0x95, 0xfa, 0x18}},
+        .expected_roots_len = 2,
+        .leaf_preimages = (uint64_t[]){0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+                                       10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
+        .preimage_count = 20,
+        .target_values = (uint64_t[]){1, 10, 16},
+        .n_target_values = 3,
+    },
+    {
+        .expected_roots =
+            {{0xe4, 0x67, 0x6b, 0xa6, 0x3c, 0x94, 0x58, 0x8b, 0xcb, 0x40, 0x83,
+              0xe5, 0x47, 0x40, 0x63, 0xb6, 0x33, 0xaa, 0x63, 0x18, 0x97, 0xdd,
+              0xaa, 0xbd, 0xe7, 0xa4, 0xda, 0x45, 0x93, 0x68, 0x90, 0xcb},
+             {0xca, 0xe9, 0x21, 0xbb, 0xf6, 0x49, 0xd4, 0xdd, 0x84, 0xc2, 0x52,
+              0xc5, 0x5c, 0x54, 0x0e, 0x2e, 0x30, 0xc6, 0xc0, 0x0c, 0xb0, 0x89,
+              0xd9, 0x70, 0x4b, 0xd6, 0x13, 0xec, 0xea, 0x30, 0x86, 0x43},
+             {0x55, 0xd0, 0xa0, 0xef, 0x8f, 0x5c, 0x25, 0xa9, 0xda, 0x26, 0x6b,
+              0x36, 0xc0, 0xc5, 0xf4, 0xb3, 0x10, 0x08, 0xec, 0xe8, 0x2d, 0xf2,
+              0x51, 0x2c, 0x89, 0x66, 0xbd, 0xdc, 0xc2, 0x7a, 0x66, 0xa0},
+             {0x4d, 0x7b, 0x3e, 0xf7, 0x30, 0x0a, 0xcf, 0x70, 0xc8, 0x92, 0xd8,
+              0x32, 0x7d, 0xb8, 0x27, 0x2f, 0x54, 0x43, 0x4a, 0xdb, 0xc6, 0x1a,
+              0x4e, 0x13, 0x0a, 0x56, 0x3c, 0xb5, 0x9a, 0x0d, 0x0f, 0x47}},
+        .expected_roots_len = 4,
+        .leaf_preimages =
+            (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
+        .preimage_count = 15,
+        .target_values = (uint64_t[]){1, 10, 5},
+        .n_target_values = 3,
+    },
+    {
+        .expected_roots =
+            {{0x35, 0x2d, 0x0d, 0x5e, 0x17, 0x23, 0x16, 0xe7, 0x3b, 0x6d, 0x0d,
+              0x40, 0x60, 0x5a, 0xd4, 0x11, 0xc7, 0x9b, 0x7e, 0x7c, 0x1e, 0xd0,
+              0xb4, 0x52, 0x9c, 0x56, 0x5f, 0x08, 0x05, 0x7b, 0xc4, 0xa8},
+             {0xca, 0xe9, 0x21, 0xbb, 0xf6, 0x49, 0xd4, 0xdd, 0x84, 0xc2, 0x52,
+              0xc5, 0x5c, 0x54, 0x0e, 0x2e, 0x30, 0xc6, 0xc0, 0x0c, 0xb0, 0x89,
+              0xd9, 0x70, 0x4b, 0xd6, 0x13, 0xec, 0xea, 0x30, 0x86, 0x43},
+             {0x55, 0xd0, 0xa0, 0xef, 0x8f, 0x5c, 0x25, 0xa9, 0xda, 0x26, 0x6b,
+              0x36, 0xc0, 0xc5, 0xf4, 0xb3, 0x10, 0x08, 0xec, 0xe8, 0x2d, 0xf2,
+              0x51, 0x2c, 0x89, 0x66, 0xbd, 0xdc, 0xc2, 0x7a, 0x66, 0xa0},
+             {0x4d, 0x7b, 0x3e, 0xf7, 0x30, 0x0a, 0xcf, 0x70, 0xc8, 0x92, 0xd8,
+              0x32, 0x7d, 0xb8, 0x27, 0x2f, 0x54, 0x43, 0x4a, 0xdb, 0xc6, 0x1a,
+              0x4e, 0x13, 0x0a, 0x56, 0x3c, 0xb5, 0x9a, 0x0d, 0x0f, 0x47}},
+        .expected_roots_len = 4,
+        .leaf_preimages =
+            (uint64_t[]){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14},
+        .preimage_count = 15,
+        .target_values = (uint64_t[]){10, 5, 16},
+        .n_target_values = 3,
+    }};
 
 void test_deletion_cases() {
   deletion_test_data *test_case;
@@ -461,7 +463,7 @@ int main() {
   test_grab_node();
   test_delete_some();
   test_deletion_cases();
-  //test_delete_with_map();
+  test_delete_with_map();
 
   return 0;
 }
diff --git a/src/leaf_map.h b/src/leaf_map.h
@@ -8,11 +8,11 @@
  * *leaf is a pointer to that leaf inside the accumulator.
  *
  * When nodes are added and removed, it's only a pointer operation, and no data
- * gets moved arround, therefore, it's fair to keep pointers and derreference
- * them to get a (undeleted) node.
+ * gets moved around, therefore, it's fair to keep pointers and dereference
+ * them to get an (undeleted) node.
  *
  * This is a simple disk-based universal hashing hash map, we allocate a
- * gigantic file at the beggining (64GB) but use a sparse file, where we
+ * gigantic file at the beginning (64GB) but use a sparse file, where we
  * "pretend" we have 64GB, but the OS doesn't allocate that until we actually
  * use the space. This file starts with zero bytes and grows as we go.
  */
@@ -21,8 +21,16 @@
 
 #include "forest_node.h"
 
+/* Represents the offset of a leaf inside the file */
+typedef unsigned long leaf_offset;
+/* The hash function we'll use to hash keys */
+typedef leaf_offset (*hashfp)(unsigned char *key);
+
+/* Our leaf map, it's a simple hash map that maps leaf hashes to leaf pointers
+ */
 typedef struct {
   int fd;
+  hashfp hash;
 } utreexo_leaf_map;
 
 /* Creates a new leaf_map. This function doesn't allocate any memory, since
@@ -31,12 +39,13 @@ typedef struct {
  */
 static inline void utreexo_leaf_map_new(utreexo_leaf_map *map,
                                         const char *filename,
-                                        const unsigned int flags);
+                                        const unsigned int flags,
+                                        const hashfp hash);
 
 /* Gets a node's reference from the map. You should pass a pointer to a pointer
  * to a utreexo_forest_node. That's because you'll end-up with a
  * utreexo_forest_node*, the actual thing is inside the mmap-ed file, taking it
- * by value would create a copy that isn't what you whant.
+ * by value would create a copy that isn't what you want.
  */
 static inline void utreexo_leaf_map_get(utreexo_leaf_map *map,
                                         utreexo_forest_node **node,