Big refactor to make PG modular

README.md

@@ -11,6 +11,9 @@ Personal-Graph is a Python library for creating, managing, and querying knowledg
 - 💬 **Natural Language Interfaces**: Natural language queries powered by Sqlite-vss and Instructor
 - 🤖 **ML-Ready**: Export data for machine learning libraries like Networkx and PyG
 
+- ✅ Supports local execution with Ollama LLMs and Embedding Models
+- ✅ Supports local db for both storing graph and embeddings
+
 ## Installation
 
 Install Personal-Graph using pip:
@@ -20,26 +23,45 @@ pip install personal-graph
 
 ## Usage
 
+### Usage of Vector store classes
+There are two vector store classes SQLiteVSS and VLiteDatabase. Let's dive down into the implementation of both the vector store classes.
+
+
+```
+from personal_graph.database import SQLiteVSS, DBClient, TursoDB
+from personal_graph import EmbeddingClient
+
+vector_store = VliteVSS()
+vector_store = VLiteDatabase(collection="memories", model_name="mxbai")
+```
+
 ### Building a Working Memory for an AI
 
 ```python
-from personal_graph import Graph
+from personal_graph import Graph, LLMClient
+from personal_graph.database import DBClient, TursoDB
+from personal_graph.vector_store import SQLiteVSS
+from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.text import text_to_graph
 
-graph = Graph("your_db_url", "your_auth_token")
+graph = Graph(vector_store=vector_store)
 
 # Insert information into the graph
-graph.insert(text="Alice is Bob's sister. Bob works at Google.")
+g = text_to_graph("Alice is Bob's sister. Bob works at Google.")
+graph.insert_graph(g)
 
 # Retrieve relevant information from the graph
 query = "Who is Alice?"
 results = graph.search(query)
+print(results["body"])
 
 # Use the retrieved information to answer questions
 print(f"Question: {query}")
 print(f"Answer: Alice is Bob's sister.")
 
 query = "Where does Bob work?"
 results = graph.search(query)
+print(results["body"])
 print(f"Question: {query}")
 print(f"Answer: Bob works at Google.")
 ```
@@ -48,9 +70,9 @@ In this example, we insert information about Alice and Bob into the knowledge gr
 
 ### Building Long-Term Memory
 ```python
-from personal_graph import Graph
+from personal_graph import Graph, LLMClient
 
-graph = Graph("your_db_url", "your_auth_token")
+graph = Graph(vector_store=vector_store)
 
 # Insert information about conversations with the user over time
 graph.insert(
@@ -88,10 +110,9 @@ graph.insert(
 # User queries about the deepest conversation
 query = "What was the deepest conversation we've ever had?"
 
-results = graph.search(query, sort_by="depth_score", descending=True, limit=1)
+deepest_conversation = graph.search(query, sort_by="depth_score", descending=True, limit=1)
 
-if results:
-    deepest_conversation = results[0]
+if deepest_conversation:
     print(f"Question: {query}")
     print(f"Answer: Our deepest conversation was on {deepest_conversation['date']} when we discussed {deepest_conversation['topic']}.")
 else:
@@ -108,27 +129,31 @@ This example demonstrates how Personal-Graph can be used to build long-term memo
 
 ### Creating and Querying a Knowledge Graph
 ```py
-from personal_graph import Graph, natural
+from personal_graph import GraphDB, LLMClient
+from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.text import text_to_graph
 
-graph = Graph("your_db_url", "your_auth_token")
+graphdb = GraphDB(vector_store=vector_store)
 
 nl_query = "Increased thirst, weight loss, increased hunger, and frequent urination are all symptoms of diabetes."
-graph = natural.insert(text=nl_query)
+graphdb = graph.insert_into_graph(text=nl_query)
 
 search_query = "I am losing weight too frequently."
-knowledge_graph = natural.search_from_graph(search_query)
+#knowledge_graph = graph.search_from_graph(search_query)
+g = text_to_graph(search_query)
+print(g)
 
-print(knowledge_graph)
+graphdb.insert_graph(g)
 ```
 
 ### Retrieval and Question-Answering
 ```py
 import dspy
-from personal_graph.graph import Graph
-from personal_graph.retriever import PersonalRM
+from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph import Graph, LLMClient, PersonalRM
 
-graph = Graph("your_db_url", "your_auth_token")
-retriever = PersonalRM(graph=graph, k=2)
+db = GraphDB() # storage_db is in-memory sqlite, vector_db is in vlite
+retriever = PersonalRM(db=db, k=2)
 
 class RAG(dspy.Module):
     def __init__(self, depth=3):
@@ -142,8 +167,62 @@ class RAG(dspy.Module):
         return dspy.Prediction(context=context, answer=prediction.answer)
 
 rag = RAG(depth=2)
-answer = rag("How is Jack related to James?")
-print(answer)
+response = rag("How is Jack related to James?")
+print(response.answer)
+```
+
+### Local Usage
+
+```py
+from personal_graph.graph import GraphDB, OllamaClient, OllamaEmbeddingClient
+from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.database import DBClient, Sqlite
+from personal_graph.vector_store import VliteVectorDB
+
+phi3 = OllamaClient(model_name="phi3")
+nomic_embed = OllamaEmbeddingClient(model_name="nomic-embed-text")
+
+storage_db = Sqlite(path="./local.db")
+vector_store = VliteVectorDB(path="./vectors")
+
+graph_generator=InstructorGraphGenerator(llm_client=phi3)
+print(graph_generator) # Should print the InstructorGraphGenerator 
+
+with GraphDB(
+    db=db, 
+    vector_store=vector_store, 
+    graph_generator=graph_generator
+  ) as db:
+    print(db)
+
+  # Graph(
+  #   db = Sqlite()
+  #   vector_store = SQLiteVSS(
+  #       persistence_layer=SQLite(
+  #           db_client=DBClient(
+  #               use_in_memory=True,
+  #               vector0_so_path="path/to/vector0.so",
+  #               vss0_so_path="path/to/vss0.so"
+  #           ),
+  #           embedding_model_client=EmbeddingClient(),
+  #       )
+  #   )
+  # )
+```
+
+### PersonalGraph to PyG, then back to PersonalGraph
+The following is just a sketch of the planned flow. WIP.
+
+```py
+graphdb = GraphDB(storage=db, vector_store=vector_store, graph_generator=graph_generator)
+
+graphdb.load_dataset("KarateClub")
+
+pyg_graph = graphdb.to_pyg()
+
+updated_graph = model(pyg_graph) # Run Neural Network algorithms here using PyG
+
+graphdb.from_pyg(updated_graph)
 ```
 
 For more details and API documentation, see the Personal-Graph Documentation.

examples/dspy_program.py

@@ -1,9 +1,20 @@
 import os
 import dspy  # type: ignore
-from personal_graph.graph import Graph
-from personal_graph.retriever import PersonalRM
-
-graph = Graph(os.getenv("LIBSQL_URL"), os.getenv("LIBSQL_AUTH_TOKEN"))
+from personal_graph import Graph, LLMClient, PersonalRM, EmbeddingClient
+from personal_graph.database import TursoDB, SQLiteVSS
+from personal_graph.graph_generator import InstructorGraphGenerator
+
+vector_store = SQLiteVSS(
+    persistence_layer=TursoDB(
+        url=os.getenv("LIBSQL_URL"), auth_token=os.getenv("LIBSQL_AUTH_TOKEN")
+    ),
+    embedding_model_client=EmbeddingClient(),
+)
+
+graph = Graph(
+    vector_store=vector_store,
+    graph_generator=InstructorGraphGenerator(llm_client=LLMClient()),
+)
 turbo = dspy.OpenAI(model="gpt-3.5-turbo", api_key=os.getenv("OPEN_API_KEY"))
 retriever = PersonalRM(graph=graph, k=2)
 dspy.settings.configure(lm=turbo, rm=retriever)
@@ -33,4 +44,5 @@ def forward(self, question):
 
 rag = RAG(depth=2)
 
-rag("How is Jack related to James?")
+response = rag("How is Jack related to James?")
+print(response.answer)