From a24d6219f218e3cd9adf853c93c9c872ef39661f Mon Sep 17 00:00:00 2001 From: Caren Thomas Date: Fri, 24 Jan 2025 16:17:21 -0800 Subject: [PATCH 1/5] chore: unit tests --- tests/custom.test.ts | 343 ++++++++++++++++++++++++------------------- 1 file changed, 195 insertions(+), 148 deletions(-) diff --git a/tests/custom.test.ts b/tests/custom.test.ts index 2ba099c..5e81e74 100644 --- a/tests/custom.test.ts +++ b/tests/custom.test.ts @@ -1,202 +1,249 @@ -// @ts-nocheck import { LettaEnvironment } from "../src/environments"; import { LettaClient } from "../src/Client"; -import { AgentState, LettaUsageStatistics, UserMessageOutput } from "../src/api"; -import { LettaStreamingResponse, MessagesListResponseItem } from "../src/api/resources/agents"; - -const client = new LettaClient({ - environment: LettaEnvironment.LettaCloud, - token: process.env.LETTA_API_KEY ?? "", -}); - -const globalAgentTracker = { - ids: new Set(), -}; - -async function createAndVerifyAgent(createOptions: Parameters[0]): Promise { - const agent = await client.agents.create(createOptions); - - expect(agent).toBeDefined(); - globalAgentTracker.ids.add(agent.id); - - const agents = await client.agents.list(); - expect(agents.some((a: AgentState) => a.id === agent.id)).toBe(true); - - return agent; -} +import { + AgentState, + SystemMessage, + UserMessage, + ToolCallMessage, + ToolReturnMessage, + AssistantMessage, +} from "../src/api"; + +describe("Letta Client", () => { + it("should create multiple agent with shared memory", async () => { + // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) + const client = new LettaClient({ + environment: LettaEnvironment.SelfHosted, + token: process.env.LETTA_API_KEY ?? "", + }); -async function cleanupAllAgents() { - for (const agentId of globalAgentTracker.ids) { - try { - await client.agents.delete(agentId); - } catch (error) { - console.error(`Failed to delete agent ${agentId}:`, error); - } - } + // Create shared memory block + let block = await client.blocks.create({ + value: "name: caren", + label: "human", + }); - globalAgentTracker.ids.clear(); -} + // Create agents and attach block + const agent1 = await client.agents.create({ + model: "openai/gpt-4", + embedding: "openai/text-embedding-ada-002", + }); + client.agents.coreMemory.attachBlock(agent1.id, block.id!); -afterAll(async () => { - await cleanupAllAgents(); -}); + const agent2 = await client.agents.create({ + model: "openai/gpt-4", + embedding: "openai/text-embedding-ada-002", + }); + client.agents.coreMemory.attachBlock(agent2.id, block.id!); -describe.skip("Create agent", () => { - it("should create an agent from default parameters", async () => { - const agent = await createAndVerifyAgent({ - memoryBlocks: [ + await client.agents.messages.create(agent1.id, { + messages: [ { - value: "username: caren", - label: "human", + role: "user", + content: "Actually, my name is Sarah.", }, ], - llmConfig: { - model: "gpt-4", - modelEndpointType: "openai", - modelEndpoint: "https://api.openai.com/v1", - contextWindow: 8192, - }, - embeddingConfig: { - embeddingModel: "text-embedding-ada-002", - embeddingEndpointType: "openai", - embeddingEndpoint: "https://api.openai.com/v1", - embeddingDim: 1536, - embeddingChunkSize: 300, - }, }); - }); - it("should create an agent with handle", async () => { - const agent = await createAndVerifyAgent({ - memoryBlocks: [ + // Validate memory has been updated for agent2 + block = await client.blocks.retrieve(block.id!); + expect(block.value.toLowerCase()).toContain("sarah"); + + block = await client.agents.coreMemory.retrieveBlock(agent2.id, "human"); + expect(block.value.toLowerCase()).toContain("sarah"); + + // Ask agent to confirm memory update + const response = await client.agents.messages.create(agent2.id, { + messages: [ { - value: "username: caren", - label: "human", + role: "user", + content: "What's my name?", }, ], - llm: "openai/gpt-4", + }); + + // Validate send message response contains new name + expect(((response.messages[0] as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); + + // Delete agents + await client.agents.delete(agent1.id); + await client.agents.delete(agent2.id); + }, 100000); + + it("create agent with custom tool", async () => { + // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) + const client = new LettaClient({ + environment: LettaEnvironment.SelfHosted, + token: process.env.LETTA_API_KEY ?? "", + }); + + const agent = await client.agents.create({ + model: "openai/gpt-4", embedding: "openai/text-embedding-ada-002", }); - }); - it("should create an agent from template", async () => { - const agent = await createAndVerifyAgent({ - memoryBlocks: [ - { - value: "username: caren", - label: "human", - }, - ], - fromTemplate: "fern-testing:latest", + const custom_tool_source_code = ` +def custom_tool(): + """Return a greeting message.""" + return "Hello world!" + `.trim(); + + const tool = await client.tools.create({ + sourceCode: custom_tool_source_code, }); - }); -}); -describe.skip("Delete agent", () => { - it("should delete an agent successfully", async () => { - const agent = await createAndVerifyAgent({ - memoryBlocks: [ + await client.agents.tools.attach(agent.id, tool.id!); + + const response = await client.agents.messages.create(agent.id, { + messages: [ { - value: "username: caren", - label: "human", + role: "user", + content: "Run custom tool and tell me what it returns", }, ], - llm: "openai/gpt-4", - embedding: "openai/text-embedding-ada-002", }); - await client.agents.delete(agent.id); - globalAgentTracker.ids.delete(agent.id); + // Validate send message response contains expected return value + expect(response.messages).toHaveLength(1); + expect(((response.messages[0] as AssistantMessage).content as string).toLowerCase()).toContain("hello world"); + }, 100000); - const agents = await client.agents.list(); - expect(agents.some((a: AgentState) => a.id === agent.id)).toBe(false); - }); -}); + it("should create single agent and send messages", async () => { + // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) + const client = new LettaClient({ + environment: LettaEnvironment.LettaCloud, + token: process.env.LETTA_API_KEY ?? "", + }); -describe.skip("Send message", () => { - it("Should send a message", async () => { - const agent = await createAndVerifyAgent({ + // Create agent with basic memory block + const agent = await client.agents.create({ memoryBlocks: [ { - value: "username: caren", + value: "name: caren", label: "human", }, ], - llm: "openai/gpt-4", + model: "openai/gpt-4", embedding: "openai/text-embedding-ada-002", }); - const messageText = "Hello, how are you today?"; + + // Validate agent persistence + let agents = await client.agents.list(); + expect(agents.some((a: AgentState) => a.id === agent.id)).toBe(true); + let messages = await client.agents.messages.list(agent.id); + expect(messages.length).toBeGreaterThan(0); + + // Send greeting message + let messageText = "Hello, how are you today?"; const response = await client.agents.messages.create(agent.id, { messages: [ { role: "user", - text: messageText, + content: messageText, }, ], }); - expect(response.messages).toHaveLength(3); + // Validate send message response contains single assistant message expect(response.usage.stepCount).toEqual(1); - expect(response.messages.map((message) => (message as { messageType?: string }).messageType)).toEqual([ - "reasoning_message", - "tool_call_message", - "tool_return_message", - ]); + expect(response.messages).toHaveLength(1); + expect(response.messages[0]).toHaveProperty("messageType", "assistant_message"); - const messages = await client.agents.messages.list(agent.id); - expect(messages.length).toBeGreaterThan(0); - const lastUserMessage = [...messages] - .reverse() - .find( - (message) => (message as MessagesListResponseItem).messageType === "user_message" - ) as UserMessageOutput; - expect(lastUserMessage).toBeDefined(); - expect(lastUserMessage?.message).toContain(messageText); - }, 10000); - - it("Should send a streaming message", async () => { - const agent = await createAndVerifyAgent({ - memoryBlocks: [ - { - value: "username: caren", - label: "human", - }, - ], - llm: "openai/gpt-4", - embedding: "openai/text-embedding-ada-002", - }); - const messageText = "Hello, how are you today?"; - const response = await client.agents.messages.stream(agent.id, { + // Validate message history + let cursor = messages[messages.length - 1].id; + messages = await client.agents.messages.list(agent.id, { after: cursor }); + expect(messages).toHaveLength(3); + + // 1. User message that was just sent + expect(messages[0]).toHaveProperty("messageType", "user_message"); + expect((messages[0] as UserMessage).content).toContain(messageText); + + // 2. Tool call for sending the assistant message back + expect(messages[1]).toHaveProperty("messageType", "tool_call_message"); + expect((messages[1] as ToolCallMessage).toolCall.name).toEqual("send_message"); + + // 3. Tool return message that contains success/failure of tool call + expect(messages[2]).toHaveProperty("messageType", "tool_return_message"); + expect((messages[2] as ToolReturnMessage).status).toEqual("success"); + + // Send message with streaming + messageText = "Actually, my name is Sarah."; + const streamResponse = await client.agents.messages.createStream(agent.id, { messages: [ { role: "user", - text: messageText, + content: messageText, }, ], }); - const responses: LettaStreamingResponse[] = []; - for await (const chunk of response) { - responses.push(chunk); + // Validate streaming response + for await (const chunk of streamResponse) { + switch (chunk.messageType) { + // 1. Reasoning message with the agent's internal monologue + case "reasoning_message": + expect(chunk.reasoning.toLowerCase()).toContain("sarah"); + break; + + // 2. Tool call to update core memory content + case "tool_call_message": + expect(chunk.toolCall.name).toEqual("core_memory_replace"); + break; + + // 3. Tool return message that contains success/failure of tool call + case "tool_return_message": + expect(chunk.status).toEqual("success"); + break; + + // 4. Assistant message that gets sent back as a reply to the original user message + case "assistant_message": + expect((chunk.content as string).toLowerCase()).toContain("sarah"); + break; + + // 5. Usage statistics message for the interaction capturing token and step count + case "usage_statistics": + expect(chunk.stepCount).toEqual(2); + break; + + default: + throw new Error(`Unexpected message type: ${chunk.messageType}`); + } } - expect(responses).toHaveLength(4); - expect((responses.pop() as LettaUsageStatistics).stepCount).toEqual(1); - expect(responses.map((message) => message.messageType)).toEqual([ - "reasoning_message", - "tool_call_message", - "tool_return_message", - ]); + // Validate message history + cursor = messages[messages.length - 1].id; + messages = await client.agents.messages.list(agent.id, { after: cursor }); + expect(messages).toHaveLength(7); - const messages = await client.agents.messages.list(agent.id); - expect(messages.length).toBeGreaterThan(0); - const lastUserMessage = [...messages] - .reverse() - .find( - (message) => (message as MessagesListResponseItem).messageType === "user_message" - ) as UserMessageOutput; - expect(lastUserMessage).toBeDefined(); - expect(lastUserMessage?.message).toContain(messageText); - }, 10000); -}); + // 1. User message that was just sent + expect(messages[0]).toHaveProperty("messageType", "user_message"); + expect((messages[0] as UserMessage).content).toContain(messageText); + + // 2. Tool call to update core memory content and send system message with update + expect(messages[1]).toHaveProperty("messageType", "tool_call_message"); + expect((messages[1] as ToolCallMessage).toolCall.name).toEqual("core_memory_replace"); + + // 3. System message with core memory update + expect(messages[2]).toHaveProperty("messageType", "system_message"); + expect(((messages[2] as SystemMessage).content as string).toLowerCase()).toContain("name: sarah"); + + // 4. Tool return message that contains success/failure of tool call + expect(messages[3]).toHaveProperty("messageType", "tool_return_message"); + expect((messages[3] as ToolReturnMessage).status).toEqual("success"); + + // 5. Tool call for sending the assistant message back + expect(messages[4]).toHaveProperty("messageType", "user_message"); + expect((messages[4] as UserMessage).content).toContain("heartbeat"); + // 6. Tool return message that contains success/failure of tool call + expect(messages[5]).toHaveProperty("messageType", "tool_call_message"); + expect((messages[5] as ToolCallMessage).toolCall.name).toEqual("send_message"); + + // 7. Tool return message that contains success/failure of tool call + expect(messages[6]).toHaveProperty("messageType", "tool_return_message"); + expect((messages[6] as ToolReturnMessage).status).toEqual("success"); + + // Delete agent + await client.agents.delete(agent.id); + }, 100000); +}); From ea291d3d3dcff5b63208deb4c57375a4833e40ba Mon Sep 17 00:00:00 2001 From: Caren Thomas Date: Fri, 24 Jan 2025 16:43:46 -0800 Subject: [PATCH 2/5] fix tool attach --- tests/custom.test.ts | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/custom.test.ts b/tests/custom.test.ts index 5e81e74..fff5c60 100644 --- a/tests/custom.test.ts +++ b/tests/custom.test.ts @@ -77,14 +77,16 @@ describe("Letta Client", () => { token: process.env.LETTA_API_KEY ?? "", }); + // Create agent const agent = await client.agents.create({ model: "openai/gpt-4", embedding: "openai/text-embedding-ada-002", }); + // Create custom tool with python source code const custom_tool_source_code = ` -def custom_tool(): - """Return a greeting message.""" +def secret_message(): + """Return a secret message.""" return "Hello world!" `.trim(); @@ -92,20 +94,40 @@ def custom_tool(): sourceCode: custom_tool_source_code, }); + // Attach custom tool to agent and invoke await client.agents.tools.attach(agent.id, tool.id!); const response = await client.agents.messages.create(agent.id, { messages: [ { role: "user", - content: "Run custom tool and tell me what it returns", + content: "Run secret message tool and tell me what it returns", }, ], }); // Validate send message response contains expected return value - expect(response.messages).toHaveLength(1); - expect(((response.messages[0] as AssistantMessage).content as string).toLowerCase()).toContain("hello world"); + expect(response.usage.stepCount).toEqual(2); + expect(response.messages).toHaveLength(3); + for (const message of response.messages) { + switch (message.messageType) { + case "tool_call_message": + expect((message as ToolCallMessage).toolCall.name).toEqual("secret_message"); + break; + case "tool_return_message": + expect((message as ToolReturnMessage).status).toEqual("success"); + break; + case "assistant_message": + expect(((message as AssistantMessage).content as string).toLowerCase()).toContain("hello world"); + break; + default: + fail(`Unexpected message type: ${(message as any).messageType}`); + } + } + + // Delete + await client.agents.delete(agent.id); + await client.tools.delete(tool.id!); }, 100000); it("should create single agent and send messages", async () => { From f8e311687b8aff4d1e54ce8682b04b7bedb993b2 Mon Sep 17 00:00:00 2001 From: Caren Thomas Date: Fri, 24 Jan 2025 17:00:34 -0800 Subject: [PATCH 3/5] add send async test --- tests/custom.test.ts | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/custom.test.ts b/tests/custom.test.ts index fff5c60..9cd9ab7 100644 --- a/tests/custom.test.ts +++ b/tests/custom.test.ts @@ -40,7 +40,7 @@ describe("Letta Client", () => { messages: [ { role: "user", - content: "Actually, my name is Sarah.", + content: "My name isn't Caren, it's Sarah. Please update your core memory with core_memory_replace", }, ], }); @@ -133,7 +133,7 @@ def secret_message(): it("should create single agent and send messages", async () => { // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) const client = new LettaClient({ - environment: LettaEnvironment.LettaCloud, + environment: LettaEnvironment.SelfHosted, token: process.env.LETTA_API_KEY ?? "", }); @@ -189,7 +189,7 @@ def secret_message(): expect((messages[2] as ToolReturnMessage).status).toEqual("success"); // Send message with streaming - messageText = "Actually, my name is Sarah."; + messageText = "My name isn't Caren, it's Sarah. Please update your core memory with core_memory_replace"; const streamResponse = await client.agents.messages.createStream(agent.id, { messages: [ { @@ -265,6 +265,43 @@ def secret_message(): expect(messages[6]).toHaveProperty("messageType", "tool_return_message"); expect((messages[6] as ToolReturnMessage).status).toEqual("success"); + // Send async message + messageText = "What's my name?"; + let run = await client.agents.messages.createAsync(agent.id, { + messages: [ + { + role: "user", + content: messageText, + }, + ], + }); + expect(run.status).toEqual("created"); + await new Promise(resolve => setTimeout(resolve, 10000)); // Sleep for 1000 ms + + run = await client.runs.retrieveRun(run.id!); + + + + // Validate send message response contains single assistant message + expect(run.status).toEqual("completed"); + const run_messages = await client.runs.listRunMessages(run.id!); + expect(run_messages).toHaveLength(3); + for (const message of run_messages) { + switch (message.messageType) { + case "user_message": + expect((message as UserMessage).content).toContain(messageText); + break; + case "assistant_message": + expect(((message as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); + break; + case "tool_return_message": + expect((message as ToolReturnMessage).status).toEqual("success"); + break; + default: + fail(`Unexpected message type: ${(message as any).messageType}`); + } + } + // Delete agent await client.agents.delete(agent.id); }, 100000); From 36a3ba5c23b1526897bad8b3afb697dca3e0f18f Mon Sep 17 00:00:00 2001 From: Caren Thomas Date: Fri, 24 Jan 2025 17:06:35 -0800 Subject: [PATCH 4/5] update --- .github/workflows/ci.yml | 6 ++-- tests/custom.test.ts | 60 ++++++++++++++++++++++++---------------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae47f90..b8766e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,8 @@ jobs: - name: Set up node uses: actions/setup-node@v3 - - name: Compile - run: yarn && yarn test - env: - LETTA_API_KEY: ${{ secrets.LETTA_API_KEY }} + - name: Test + run: yarn && yarn test --testPathIgnorePatterns tests/custom.test.ts publish: needs: [ compile, test ] diff --git a/tests/custom.test.ts b/tests/custom.test.ts index 9cd9ab7..1f0bbb5 100644 --- a/tests/custom.test.ts +++ b/tests/custom.test.ts @@ -9,34 +9,40 @@ import { AssistantMessage, } from "../src/api"; +const environment = process.env.LETTA_ENV === 'localhost' + ? LettaEnvironment.SelfHosted + : LettaEnvironment.LettaCloud; + describe("Letta Client", () => { it("should create multiple agent with shared memory", async () => { - // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) const client = new LettaClient({ - environment: LettaEnvironment.SelfHosted, + environment: environment, token: process.env.LETTA_API_KEY ?? "", }); // Create shared memory block let block = await client.blocks.create({ - value: "name: caren", + value: "name: Caren", label: "human", }); // Create agents and attach block const agent1 = await client.agents.create({ - model: "openai/gpt-4", + model: "openai/gpt-4o-mini", embedding: "openai/text-embedding-ada-002", + blockIds: [block.id!], }); - client.agents.coreMemory.attachBlock(agent1.id, block.id!); + expect(agent1.memory.blocks[0].id).toEqual(block.id); - const agent2 = await client.agents.create({ - model: "openai/gpt-4", + let agent2 = await client.agents.create({ + model: "openai/gpt-4o-mini", embedding: "openai/text-embedding-ada-002", }); - client.agents.coreMemory.attachBlock(agent2.id, block.id!); + // Another way to attach memory blocks + agent2 = await client.agents.coreMemory.attachBlock(agent2.id, block.id!); + expect(agent2.memory.blocks[0].id).toEqual(block.id); - await client.agents.messages.create(agent1.id, { + let response = await client.agents.messages.create(agent1.id, { messages: [ { role: "user", @@ -53,7 +59,7 @@ describe("Letta Client", () => { expect(block.value.toLowerCase()).toContain("sarah"); // Ask agent to confirm memory update - const response = await client.agents.messages.create(agent2.id, { + response = await client.agents.messages.create(agent1.id, { messages: [ { role: "user", @@ -68,18 +74,18 @@ describe("Letta Client", () => { // Delete agents await client.agents.delete(agent1.id); await client.agents.delete(agent2.id); + await client.blocks.delete(block.id!); }, 100000); it("create agent with custom tool", async () => { - // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) const client = new LettaClient({ - environment: LettaEnvironment.SelfHosted, + environment: environment, token: process.env.LETTA_API_KEY ?? "", }); // Create agent const agent = await client.agents.create({ - model: "openai/gpt-4", + model: "openai/gpt-4o-mini", embedding: "openai/text-embedding-ada-002", }); @@ -131,9 +137,9 @@ def secret_message(): }, 100000); it("should create single agent and send messages", async () => { - // Initialize client (to run locally, override using LettaEnvironment.SelfHosted) + console.log("start"); const client = new LettaClient({ - environment: LettaEnvironment.SelfHosted, + environment: environment, token: process.env.LETTA_API_KEY ?? "", }); @@ -168,13 +174,13 @@ def secret_message(): // Validate send message response contains single assistant message expect(response.usage.stepCount).toEqual(1); - expect(response.messages).toHaveLength(1); + expect(response.messages).toHaveLength(1); // SHOULD BE 2 WITH REASONING MESSAGE expect(response.messages[0]).toHaveProperty("messageType", "assistant_message"); // Validate message history let cursor = messages[messages.length - 1].id; messages = await client.agents.messages.list(agent.id, { after: cursor }); - expect(messages).toHaveLength(3); + expect(messages).toHaveLength(3); // SHOULD BE A REASONING MESSAGE HERE TOO // 1. User message that was just sent expect(messages[0]).toHaveProperty("messageType", "user_message"); @@ -187,6 +193,7 @@ def secret_message(): // 3. Tool return message that contains success/failure of tool call expect(messages[2]).toHaveProperty("messageType", "tool_return_message"); expect((messages[2] as ToolReturnMessage).status).toEqual("success"); + console.log("done 1"); // Send message with streaming messageText = "My name isn't Caren, it's Sarah. Please update your core memory with core_memory_replace"; @@ -264,6 +271,7 @@ def secret_message(): // 7. Tool return message that contains success/failure of tool call expect(messages[6]).toHaveProperty("messageType", "tool_return_message"); expect((messages[6] as ToolReturnMessage).status).toEqual("success"); + console.log("done 2"); // Send async message messageText = "What's my name?"; @@ -276,25 +284,29 @@ def secret_message(): ], }); expect(run.status).toEqual("created"); - await new Promise(resolve => setTimeout(resolve, 10000)); // Sleep for 1000 ms + // Wait for run to complete + await new Promise(resolve => setTimeout(resolve, 10000)); run = await client.runs.retrieveRun(run.id!); - - - - // Validate send message response contains single assistant message expect(run.status).toEqual("completed"); + + // Validate messages from run const run_messages = await client.runs.listRunMessages(run.id!); expect(run_messages).toHaveLength(3); for (const message of run_messages) { switch (message.messageType) { + // 1. User message that was just sent case "user_message": expect((message as UserMessage).content).toContain(messageText); break; - case "assistant_message": + + // 2. Assistant message with response + case "assistant_message": // ADD REASONING expect(((message as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); break; - case "tool_return_message": + + // 3. Tool call for sending the assistant message response back + case "tool_return_message": // THIS SHOULD NOT BE RETURNED expect((message as ToolReturnMessage).status).toEqual("success"); break; default: From 647a081a296297f5099259f37c2e35cfa440c515 Mon Sep 17 00:00:00 2001 From: Caren Thomas Date: Mon, 3 Feb 2025 14:33:29 -0800 Subject: [PATCH 5/5] more updates --- .github/workflows/tests.yml | 35 ++++++++++++++ tests/custom.test.ts | 92 ++++++++++++++++++++----------------- 2 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..38771db --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,35 @@ +name: tests + +on: + push: + branches: [ main ] + pull_request: + +jobs: + compile: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up node + uses: actions/setup-node@v3 + + - name: Compile + run: yarn && yarn build + + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Set up node + uses: actions/setup-node@v3 + + - name: Test + run: yarn && yarn test tests/custom.test.ts + env: + LETTA_API_KEY: ${{ secrets.LETTA_API_KEY }} \ No newline at end of file diff --git a/tests/custom.test.ts b/tests/custom.test.ts index 1f0bbb5..cdd7d45 100644 --- a/tests/custom.test.ts +++ b/tests/custom.test.ts @@ -2,11 +2,12 @@ import { LettaEnvironment } from "../src/environments"; import { LettaClient } from "../src/Client"; import { AgentState, + AssistantMessage, + ReasoningMessage, SystemMessage, - UserMessage, ToolCallMessage, ToolReturnMessage, - AssistantMessage, + UserMessage, } from "../src/api"; const environment = process.env.LETTA_ENV === 'localhost' @@ -69,7 +70,7 @@ describe("Letta Client", () => { }); // Validate send message response contains new name - expect(((response.messages[0] as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); + expect(((response.messages[1] as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); // Delete agents await client.agents.delete(agent1.id); @@ -96,7 +97,7 @@ def secret_message(): return "Hello world!" `.trim(); - const tool = await client.tools.create({ + const tool = await client.tools.upsert({ sourceCode: custom_tool_source_code, }); @@ -114,18 +115,25 @@ def secret_message(): // Validate send message response contains expected return value expect(response.usage.stepCount).toEqual(2); - expect(response.messages).toHaveLength(3); + expect(response.messages).toHaveLength(5); for (const message of response.messages) { switch (message.messageType) { + case "reasoning_message": + expect((message as ReasoningMessage).reasoning.toLowerCase()).toContain("secret message"); + break; + case "tool_call_message": expect((message as ToolCallMessage).toolCall.name).toEqual("secret_message"); break; + case "tool_return_message": expect((message as ToolReturnMessage).status).toEqual("success"); break; + case "assistant_message": expect(((message as AssistantMessage).content as string).toLowerCase()).toContain("hello world"); break; + default: fail(`Unexpected message type: ${(message as any).messageType}`); } @@ -137,7 +145,6 @@ def secret_message(): }, 100000); it("should create single agent and send messages", async () => { - console.log("start"); const client = new LettaClient({ environment: environment, token: process.env.LETTA_API_KEY ?? "", @@ -151,7 +158,7 @@ def secret_message(): label: "human", }, ], - model: "openai/gpt-4", + model: "openai/gpt-4o-mini", embedding: "openai/text-embedding-ada-002", }); @@ -174,26 +181,24 @@ def secret_message(): // Validate send message response contains single assistant message expect(response.usage.stepCount).toEqual(1); - expect(response.messages).toHaveLength(1); // SHOULD BE 2 WITH REASONING MESSAGE - expect(response.messages[0]).toHaveProperty("messageType", "assistant_message"); + expect(response.messages).toHaveLength(2); + expect(response.messages[0]).toHaveProperty("messageType", "reasoning_message"); + expect(response.messages[1]).toHaveProperty("messageType", "assistant_message"); // Validate message history let cursor = messages[messages.length - 1].id; messages = await client.agents.messages.list(agent.id, { after: cursor }); - expect(messages).toHaveLength(3); // SHOULD BE A REASONING MESSAGE HERE TOO + expect(messages).toHaveLength(3); // 1. User message that was just sent expect(messages[0]).toHaveProperty("messageType", "user_message"); expect((messages[0] as UserMessage).content).toContain(messageText); - // 2. Tool call for sending the assistant message back - expect(messages[1]).toHaveProperty("messageType", "tool_call_message"); - expect((messages[1] as ToolCallMessage).toolCall.name).toEqual("send_message"); + // 2. Assistant message with response + expect(messages[1]).toHaveProperty("messageType", "assistant_message"); - // 3. Tool return message that contains success/failure of tool call - expect(messages[2]).toHaveProperty("messageType", "tool_return_message"); - expect((messages[2] as ToolReturnMessage).status).toEqual("success"); - console.log("done 1"); + // 3. Reasoning message with agent's internal monologue + expect(messages[2]).toHaveProperty("messageType", "reasoning_message"); // Send message with streaming messageText = "My name isn't Caren, it's Sarah. Please update your core memory with core_memory_replace"; @@ -211,7 +216,6 @@ def secret_message(): switch (chunk.messageType) { // 1. Reasoning message with the agent's internal monologue case "reasoning_message": - expect(chunk.reasoning.toLowerCase()).toContain("sarah"); break; // 2. Tool call to update core memory content @@ -242,7 +246,10 @@ def secret_message(): // Validate message history cursor = messages[messages.length - 1].id; messages = await client.agents.messages.list(agent.id, { after: cursor }); - expect(messages).toHaveLength(7); + if (messages.length > 0 && messages[0].messageType == "tool_return_message") { + messages.shift(); + } + expect(messages).toHaveLength(8); // 1. User message that was just sent expect(messages[0]).toHaveProperty("messageType", "user_message"); @@ -252,26 +259,27 @@ def secret_message(): expect(messages[1]).toHaveProperty("messageType", "tool_call_message"); expect((messages[1] as ToolCallMessage).toolCall.name).toEqual("core_memory_replace"); - // 3. System message with core memory update - expect(messages[2]).toHaveProperty("messageType", "system_message"); - expect(((messages[2] as SystemMessage).content as string).toLowerCase()).toContain("name: sarah"); + // 3. Reasoning message with core memory update + expect(messages[2]).toHaveProperty("messageType", "reasoning_message"); - // 4. Tool return message that contains success/failure of tool call - expect(messages[3]).toHaveProperty("messageType", "tool_return_message"); - expect((messages[3] as ToolReturnMessage).status).toEqual("success"); + // 4. System message with core memory update + expect(messages[3]).toHaveProperty("messageType", "system_message"); + expect(((messages[3] as SystemMessage).content as string).toLowerCase()).toContain("name: sarah"); - // 5. Tool call for sending the assistant message back - expect(messages[4]).toHaveProperty("messageType", "user_message"); - expect((messages[4] as UserMessage).content).toContain("heartbeat"); + // 5. Tool return message that contains success/failure of tool call + expect(messages[4]).toHaveProperty("messageType", "tool_return_message"); + expect((messages[4] as ToolReturnMessage).status).toEqual("success"); - // 6. Tool return message that contains success/failure of tool call - expect(messages[5]).toHaveProperty("messageType", "tool_call_message"); - expect((messages[5] as ToolCallMessage).toolCall.name).toEqual("send_message"); + // 6. Tool call for sending the assistant message back + expect(messages[5]).toHaveProperty("messageType", "user_message"); + expect((messages[5] as UserMessage).content).toContain("heartbeat"); - // 7. Tool return message that contains success/failure of tool call - expect(messages[6]).toHaveProperty("messageType", "tool_return_message"); - expect((messages[6] as ToolReturnMessage).status).toEqual("success"); - console.log("done 2"); + // 7. Assistant message that response + expect(messages[6]).toHaveProperty("messageType", "assistant_message"); + expect(((messages[6] as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); + + // 8. Reasoning message that contains inner monologue of agent + expect(messages[7]).toHaveProperty("messageType", "reasoning_message"); // Send async message messageText = "What's my name?"; @@ -291,7 +299,7 @@ def secret_message(): expect(run.status).toEqual("completed"); // Validate messages from run - const run_messages = await client.runs.listRunMessages(run.id!); + const run_messages = await client.runs.listRunMessages(run.id!, { order: "asc" }); expect(run_messages).toHaveLength(3); for (const message of run_messages) { switch (message.messageType) { @@ -299,16 +307,16 @@ def secret_message(): case "user_message": expect((message as UserMessage).content).toContain(messageText); break; + + // 2. Reasoning message with response + case "reasoning_message": + break; - // 2. Assistant message with response - case "assistant_message": // ADD REASONING + // 3. Assistant message with response + case "assistant_message": expect(((message as AssistantMessage).content as string).toLowerCase()).toContain("sarah"); break; - // 3. Tool call for sending the assistant message response back - case "tool_return_message": // THIS SHOULD NOT BE RETURNED - expect((message as ToolReturnMessage).status).toEqual("success"); - break; default: fail(`Unexpected message type: ${(message as any).messageType}`); }