Features/cached graphs (#1656)

* rename args * impl config loader * impl cache * add global plugins to data, rid ttl, fix gql apis and add a bunch of tests * impl loading disk graph and fix tests * fmt * impl overwrite load, fix cases with storage feature disabled, add tests, fix tests, impl save_to_path mat * more tests * fix rename graph gql * create work_dir if not found * Todos/graphql (#1672) * impl gqlgraphs * properties, unique layers to gqlgraphs * impl default configs, impl config load precedence, add tests * impl auth configs * wait for server to come online when started, also expose an api on client to see if server is already online, add tests * add client tests * fix tests * add load graphs tests * rid duplicate tests * wait internalised * rid silly auth config defaults * Todos/graphql (#1676) * remove timeout from run and set a sensible default * redo appconfig building * skip graphs that don't have name as property instead of failing * rename graphs names gql to name * remove Graphs hashmap/paths from the RaphtoryServer so all uploads are done via the client * rid expects * impl graph file upload * impl load graph from path * impl overwrite graph * impl namespace * handle more namespace validation cases, add tests * impl get_graph to provide namespace as arg, modified gqlgraphs to hold both names, and namespaces but not graphs saving memory, impl path api on both gqlgraph and gqlgraphs, fixed and added tests * refactor and fix save_graph * impl load_graph_from_path ns * fix issue with dir creation * add rename graph tests, rid saving graph path as graph property, add parent_graph_namespace to save graph and rename graph, add more validation in these apis * add tests for get graphs * add received graph tests * add test for get graph * add test for update last opened * add isarchive tests * fix save and add tests * fix save graph issue * impl create_graph, update_graph and add tests for update graph with new name * impl tests for update graph * add tests for send graph * fix upload graph to accept namespace and add tests * fix load graph and add tests * impl tests for get graph * impl tests for get graphs * fix receive graph versioning issue and add tests * impl rename graph tests * fix archive tests * fix rename graph and add comments * Change graph_nodes to be a vector of string than string, fix issues with send_graph from py and gql, receive graph versioning, refactors * rename renamegraph to movegraph and impl copy graph, add tests for move graph * add tests fro copy graph * impl delete graph gql api and add tests * add properties to nodes and edges to test if they are carried forward when creating new graph with update_graph * fix create graph and add tests * rid dependency of graph name as graph prop, fix tests * replace data String work_dir with PathBuf * namespace changes, fix tests, rid dead code * namespace changes for python, fix tests, fix apis and comments * ref * rid arc cache * review changes, fix warnings * appconf syntax sugar * review changes * impl copy graph helper * move graph helper * delete graph helper * receive graph helper * fix imports * fmt * fix imports * simplify tests * fix tests * fix tests * fmt * fix tests * fix test * fix tests * fix eviction test * fix pytest * fix test * fix tests * fix tests * fix tests * fix tests * fix test * rid load graph * rid auth and tracing * return graph as byte in python * return unit instead of bool * ref * rid auth * default timeout and rename var * fix return type * rid dead code * raise exceptions and fix all tests * return graph instead of bytes * fix warnings * fix imports * fix ci * fix upload graph return type --------- Co-authored-by: Shivam Kapoor <[email protected]>
Pometry · Aug 8, 2024 · bcc2e87 · bcc2e87
1 parent 7e0db58
commit bcc2e87
Show file tree

Hide file tree

Showing 36 changed files with 4,664 additions and 1,688 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -148,6 +148,8 @@ arrow-schema = { version = "50" }
 arrow-data = { version = "50" }
 arrow-array = { version = "50" }
 
+moka = { version = "0.12.7", features = ["sync"] }
+
 # Make sure that transitive dependencies stick to disk_graph 50
 [patch.crates-io]
 arrow = { git = "https://github.com/apache/arrow-rs.git", tag = "50.0.0" }

diff --git a/js-raphtory/src/graph/misc.rs b/js-raphtory/src/graph/misc.rs
@@ -54,7 +54,7 @@ impl From<JsProp> for JsValue {
             )
             .into(),
             Prop::Graph(v) => Graph(UnderGraph::TGraph(Arc::new(v))).into(),
-            Prop::PersistentGraph(_v) => todo!("PersistentGraph not yet implemented"),
+            Prop::PersistentGraph(_) => todo!("PersistentGraph not yet implemented"),
             Prop::List(v) => {
                 let v: Array = v.iter().map(|v| JsValue::from(JsProp(v.clone()))).collect();
                 v.into()

diff --git a/pometry-storage-private b/pometry-storage-private
diff --git a/python/Cargo.toml b/python/Cargo.toml
@@ -22,7 +22,7 @@ pyo3 = { workspace = true }
 raphtory_core = { path = "../raphtory", version = "0.10.0", features = ["python", "search", "vectors", "proto"], package = "raphtory" }
 raphtory-graphql = { path = "../raphtory-graphql", version = "0.10.0",features = ["python"] }
 serde_json = { workspace = true }
-reqwest = { workspace = true }
+reqwest = { workspace = true, features = ["multipart"] }
 tokio = { workspace = true }
 crossbeam-channel = { workspace = true }
 serde = { workspace = true }

diff --git a/python/tests/test_algorithms.py b/python/tests/test_algorithms.py
@@ -192,7 +192,6 @@ def test_algo_result():
     expected_result = pd.DataFrame({"Key": [1], "Value": [1]})
     row_with_one = df[df["Key"] == 1]
     row_with_one.reset_index(inplace=True, drop=True)
-    print(row_with_one)
     assert row_with_one.equals(expected_result)
     # Algo Str u64
     actual = algorithms.weakly_connected_components(g)

diff --git a/python/tests/test_graphdb.py b/python/tests/test_graphdb.py
@@ -2002,7 +2002,6 @@ def test_layers_earliest_time():
     g = Graph()
     e = g.add_edge(1, 1, 2, layer="test")
     e = g.edge(1, 2)
-    print(e)
     assert e.earliest_time == 1
 
 
@@ -2025,7 +2024,6 @@ def test_edge_explode_layers():
     e_layers = [ee.layer_names for ee in layered_edges]
     e_layer_prop = [[str(ee.properties["layer"])] for ee in layered_edges]
     assert e_layers == e_layer_prop
-    print(e_layers)
 
     nested_layered_edges = g.nodes.out_edges.explode_layers()
     e_layers = [[ee.layer_names for ee in edges] for edges in nested_layered_edges]
@@ -2034,11 +2032,8 @@ def test_edge_explode_layers():
         for layered_edges in nested_layered_edges
     ]
     assert e_layers == e_layer_prop
-    print(e_layers)
 
-    print(g.nodes.out_neighbours.collect)
     nested_layered_edges = g.nodes.out_neighbours.out_edges.explode_layers()
-    print(nested_layered_edges)
     e_layers = [
         [ee.layer_names for ee in layered_edges]
         for layered_edges in nested_layered_edges
@@ -2048,7 +2043,6 @@ def test_edge_explode_layers():
         for layered_edges in nested_layered_edges
     ]
     assert e_layers == e_layer_prop
-    print(e_layers)
 
 
 def test_starend_edges():

diff --git a/python/tests/test_graphdb_imports.py b/python/tests/test_graphdb_imports.py
@@ -20,10 +20,9 @@ def test_import_into_graph():
     assert res.properties.constant.get("con") == 11
 
     gg = Graph()
-    res = gg.import_nodes([g_a, g_b])
-    assert len(res) == 2
+    gg.import_nodes([g_a, g_b])
     assert len(gg.nodes) == 2
-    assert [x.name for x in res] == ["A", "B"]
+    assert [x.name for x in gg.nodes] == ["A", "B"]
 
     e_a_b = g.add_edge(2, "A", "B")
     res = gg.import_edge(e_a_b)
@@ -37,7 +36,6 @@ def test_import_into_graph():
 
     e_c_d = g.add_edge(4, "C", "D")
     gg = Graph()
-    res = gg.import_edges([e_a_b, e_c_d])
-    assert len(res) == 2
+    gg.import_edges([e_a_b, e_c_d])
     assert len(gg.nodes) == 4
     assert len(gg.edges) == 2