chore: [WIP] many demo changes, breaks

README.md

@@ -11,6 +11,9 @@ Personal-Graph is a Python library for creating, managing, and querying knowledg
 - 💬 **Natural Language Interfaces**: Natural language queries powered by Sqlite-vss and Instructor
 - 🤖 **ML-Ready**: Export data for machine learning libraries like Networkx and PyG
 
+- ✅ Supports local execution with Ollama LLMs and Embedding Models
+- ✅ Supports local db for both storing graph and embeddings
+
 ## Installation
 
 Install Personal-Graph using pip:
@@ -28,72 +31,36 @@ There are two vector store classes SQLiteVSS and VLiteDatabase. Let's dive down
 from personal_graph.database import SQLiteVSS, DBClient, TursoDB
 from personal_graph import EmbeddingClient
 
-# Using SQLiteVSS vector database and connecting with Turso DB
-vector_store = SQLiteVSS(
-        persistence_layer=TursoDB(
-            db_client=DBClient(
-                db_url=args.db_url,
-                db_auth_token=args.db_auth_token,
-            ),
-            embedding_model_client=EmbeddingClient(),
-        )
-    )
-
-# Using SQLiteVSS vector database and connecting with local DB
-vector_store = SQLiteVSS(
-        persistence_layer=SQLite(
-            db_client=DBClient(
-                use_in_memory=True,
-                vector0_so_path="path/to/vector0.so",
-                vss0_so_path="path/to/vss0.so"
-            ),
-            embedding_model_client=EmbeddingClient(),
-        )
-    )
-
-# Using SQLiteVSS vector database and connecting with in-memory DB
-vector_store = SQLiteVSS(
-        persistence_layer=SQLite(
-            db_client=DBClient(
-                use_in_memory=True,
-                vector0_so_path="path/to/vector0.so",
-                vss0_so_path="path/to/vss0.so"
-            ),
-            embedding_model_client=EmbeddingClient(),
-        )
-    )
-
-# Using VLite vector database 
-from personal_graph.database import VLiteDatabase
-
-vector_store = VLiteDatabase(collection="your-collection-name", model_name="embedding-model-name")
-
+vector_store = VliteVSS()
+vector_store = VLiteDatabase(collection="memories", model_name="mxbai")
 ```
 
 ### Building a Working Memory for an AI
 
 ```python
 from personal_graph import Graph, LLMClient
-from personal_graph.database import SQLiteVSS, DBClient, TursoDB
+from personal_graph.database import DBClient, TursoDB
+from personal_graph.vector_store import SQLiteVSS
 from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.text import text_to_graph
 
-
-graph = Graph(vector_store=vector_store, graph_generator=InstructorGraphGenerator(llm_client=LLMClient()))
+graph = Graph(vector_store=vector_store)
 
 # Insert information into the graph
-graph.insert_into_graph(text="Alice is Bob's sister. Bob works at Google.")
+g = text_to_graph("Alice is Bob's sister. Bob works at Google.")
+graph.insert_graph(g)
 
 # Retrieve relevant information from the graph
 query = "Who is Alice?"
-results = graph.search(query, limit=1)
+results = graph.search(query)
 print(results["body"])
 
 # Use the retrieved information to answer questions
 print(f"Question: {query}")
 print(f"Answer: Alice is Bob's sister.")
 
 query = "Where does Bob work?"
-results = graph.search(query, limit=1)
+results = graph.search(query)
 print(results["body"])
 print(f"Question: {query}")
 print(f"Answer: Bob works at Google.")
@@ -104,9 +71,8 @@ In this example, we insert information about Alice and Bob into the knowledge gr
 ### Building Long-Term Memory
 ```python
 from personal_graph import Graph, LLMClient
-from personal_graph.graph_generator import InstructorGraphGenerator
 
-graph = Graph(vector_store=vector_store, graph_generator=InstructorGraphGenerator(llm_client=LLMClient()))
+graph = Graph(vector_store=vector_store)
 
 # Insert information about conversations with the user over time
 graph.insert(
@@ -163,18 +129,21 @@ This example demonstrates how Personal-Graph can be used to build long-term memo
 
 ### Creating and Querying a Knowledge Graph
 ```py
-from personal_graph import Graph, LLMClient
+from personal_graph import GraphDB, LLMClient
 from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.text import text_to_graph
 
-graph = Graph(vector_store=vector_store, graph_generator=InstructorGraphGenerator(llm_client=LLMClient()))
+graphdb = GraphDB(vector_store=vector_store)
 
 nl_query = "Increased thirst, weight loss, increased hunger, and frequent urination are all symptoms of diabetes."
-graph = graph.insert_into_graph(text=nl_query)
+graphdb = graph.insert_into_graph(text=nl_query)
 
 search_query = "I am losing weight too frequently."
-knowledge_graph = graph.search_from_graph(search_query)
+#knowledge_graph = graph.search_from_graph(search_query)
+g = text_to_graph(search_query)
+print(g)
 
-print(knowledge_graph)
+graphdb.insert_graph(g)
 ```
 
 ### Retrieval and Question-Answering
@@ -183,8 +152,8 @@ import dspy
 from personal_graph.graph_generator import InstructorGraphGenerator
 from personal_graph import Graph, LLMClient, PersonalRM
 
-graph = Graph(vector_store=vector_store, graph_generator=InstructorGraphGenerator(llm_client=LLMClient()))
-retriever = PersonalRM(graph=graph, k=2)
+db = GraphDB() # storage_db is in-memory sqlite, vector_db is in vlite
+retriever = PersonalRM(db=db, k=2)
 
 class RAG(dspy.Module):
     def __init__(self, depth=3):
@@ -202,17 +171,58 @@ response = rag("How is Jack related to James?")
 print(response.answer)
 ```
 
-### Providing Custom LLMClient and EmbeddingClient
+### Local Usage
 
 ```py
-from personal_graph.graph import Graph, LLMClient, EmbeddingClient
+from personal_graph.graph import GraphDB, OllamaClient, OllamaEmbeddingClient
 from personal_graph.graph_generator import InstructorGraphGenerator
+from personal_graph.database import DBClient, Sqlite
+from personal_graph.vector_store import VliteVectorDB
+
+phi3 = OllamaClient(model_name="phi3")
+nomic_embed = OllamaEmbeddingClient(model_name="nomic-embed-text")
+
+storage_db = Sqlite(path="./local.db")
+vector_store = VliteVectorDB(path="./vectors")
+
+graph_generator=InstructorGraphGenerator(llm_client=phi3)
+print(graph_generator) # Should print the InstructorGraphGenerator 
+
+with GraphDB(
+    db=db, 
+    vector_store=vector_store, 
+    graph_generator=graph_generator
+  ) as db:
+    print(db)
+
+  # Graph(
+  #   db = Sqlite()
+  #   vector_store = SQLiteVSS(
+  #       persistence_layer=SQLite(
+  #           db_client=DBClient(
+  #               use_in_memory=True,
+  #               vector0_so_path="path/to/vector0.so",
+  #               vss0_so_path="path/to/vss0.so"
+  #           ),
+  #           embedding_model_client=EmbeddingClient(),
+  #       )
+  #   )
+  # )
+```
+
+### PersonalGraph to PyG, then back to PersonalGraph
+The following is just a sketch of the planned flow. WIP.
+
+```py
+graphdb = GraphDB(storage=db, vector_store=vector_store, graph_generator=graph_generator)
+
+graphdb.load_dataset("KarateClub")
 
-your_litellm_client = LLMClient(client="llm-client", model_name="llm-model-name")
-your_embedding_client = EmbeddingClient(client="embedding-client", model_name="embedding-model-name", dimensions="model-dimension")
+pyg_graph = graphdb.to_pyg()
 
-graph = Graph(vector_store=vector_store, graph_generator=InstructorGraphGenerator(llm_client=your_litellm_client))
+updated_graph = model(pyg_graph) # Run Neural Network algorithms here using PyG
 
+graphdb.from_pyg(updated_graph)
 ```
 
 For more details and API documentation, see the Personal-Graph Documentation.

personal_graph/clients.py

@@ -4,38 +4,72 @@
 from abc import ABC, abstractmethod
 
 
-class OpenAIClient(ABC):
-    def __init__(self, client=None) -> None:
-        if client is None:
-            client = self._create_default_client()
-        self.client = client
+class APIClient(ABC):
+    # def __init__(self, client=None) -> None:
+    #     if client is None:
+    #         client = self._create_default_client()
+    #     self.client = client
 
     @abstractmethod
     def _create_default_client(self):
         pass
 
 
-class EmbeddingClient(OpenAIClient):
-    def __init__(
-        self, client=None, model_name="openai/text-embedding-3-small", dimensions=384
-    ) -> None:
-        super().__init__(client)
-        self.model_name = model_name
-        self.dimensions = dimensions
+class EmbeddingModel(ABC):
+    # def __init__(self, client=None) -> None:
+    #     if client is None:
+    #         client = self._create_default_client()
+    #     self.client = client
+
+    @abstractmethod
+    def _create_default_client(self):
+        pass
+
+
+@dataclasses
+class OpenAIEmbeddingModel(EmbeddingModel):
+    model_name: str = "text-embedding-3-small"
+    dimensions: Optional[int] = 384
+    api_key: str = ""
+
+    def __post_init__(self):
+        return openai.OpenAI(
+            api_key=os.getenv("OPENAI_API_KEY", self.api_key)
+        )
+
+@dataclasses
+class LiteLLMEmbeddingClient(APIClient):
+    model_name: str = "text-embedding-3-small"
+    dimensions: Optional[int] = 384
+    base_url: str = ""
 
     def _create_default_client(self):
         return openai.OpenAI(
             api_key="",
-            base_url=os.getenv("LITE_LLM_BASE_URL", ""),
+            base_url=os.getenv("LITE_LLM_BASE_URL", self.base_url),
             default_headers={
                 "Authorization": f"Bearer {os.getenv('LITE_LLM_TOKEN', '')}"
             },
         )
 
+@dataclasses
+class OllamaEmbeddingClient(APIClient):
+    model_name: str = "text-embedding-3-small"
+    dimensions: Optional[int]
+    # def __init__(
+    #     self, client=None, model_name="openai/text-embedding-3-small", dimensions=384
+    # ) -> None:
+    #     super().__init__(client)
+    #     self.model_name = model_name
+    #     self.dimensions = dimensions
+
+    def _create_default_client(self):
+        return ollama.Ollama()
+
 
-class LLMClient(OpenAIClient):
+class OpenAILLMClient(APIClient):
     def __init__(self, client=None, model_name="openai/gpt-3.5-turbo") -> None:
-        super().__init__(client)
+        # super().__init__(client)
         self.model_name = model_name
 
     def _create_default_client(self):

personal_graph/graph_generator/generator.py

@@ -5,30 +5,28 @@
 from personal_graph.models import KnowledgeGraph
 
 
-class GraphGenerator(ABC):
+class TextToGraphParserInterface(ABC):
     @abstractmethod
     def generate(self, query: str) -> KnowledgeGraph:
         """Generate a KnowledgeGraph from the given query."""
         pass
 
 
-class InstructorGraphGenerator(GraphGenerator):
+class OpenAITextToGraphParser(TextToGraphParserInterface):
     def __init__(
         self,
         llm_client: LLMClient,
-        function_calling_model: str = "gpt-3.5-turbo",
         system_prompt: str = "You are a high quality knowledge graph generator based on the user query for the purpose of generating descriptive, informative, detailed and accurate knowledge graphs. You can generate proper nodes and edges as a knowledge graph.",
         prompt: str = "Help me describe this user query as a detailed knowledge graph with meaningful relationships that should provide some descriptive attributes(attribute is the detailed and proper information about the edge) and informative labels about the nodes and relationship. Try to make most of the relationships between similar nodes.",
     ):
-        self.function_calling_model = function_calling_model
         self.system_prompt = system_prompt
         self.prompt = prompt
         self.llm_client = llm_client
 
     def generate(self, query: str) -> KnowledgeGraph:
         client = instructor.from_openai(self.llm_client.client)
         knowledge_graph = client.chat.completions.create(
-            model=self.function_calling_model,
+            model=self.llm_client.model_name,
             messages=[
                 {
                     "role": "system",

personal_graph/persistence_layer/database/postgres/README.md

@@ -0,0 +1 @@
+# TODO

personal_graph/persistence_layer/database/redis/README.md

@@ -0,0 +1 @@
+# TODO