feat: BDD and evaluation framework (#101) (#137)

The idea is that the BDD valuation takes place after every iteration, which is a little bit suboptimal but, on the other hand, lets the users control the entire flow (and, for example, break it if something goes unexpected). Related to: #101
callstackincubator · Dec 20, 2024 · c9bb269 · c9bb269
1 parent 66bc887
commit c9bb269
Show file tree

Hide file tree

Showing 35 changed files with 1,394 additions and 405 deletions.
diff --git a/README.md b/README.md
@@ -39,6 +39,7 @@ Built with TypeScript and designed to be serverless-ready.
   - [Completing the workflow](#completing-the-workflow)
   - [Long-running operations](#long-running-operations)
   - [Custom execution](#custom-execution)
+- [Test framework](./packages/bdd/README.md)
 - [Contributors](#contributors)
 - [Made with ❤️ at Callstack](#made-with-❤️-at-callstack)
 
@@ -426,6 +427,11 @@ If you want to handle tool execution manually, you can use `iterate` function to
 
 Have a look at how `teamwork` is implemented [here](./packages/framework/src/teamwork.ts) to understand how it works.
 
+
+### BDD Testing
+
+There's a packaged called `fabrice-ai/bdd` dedicated to unit testing - actually to Behavioral Driven Development. [Check the docs](./packages/bdd/README.md).
+
 ## Contributors
 
 <!-- ALL-CONTRIBUTORS-LIST:START -->

diff --git a/bun.lockb b/bun.lockb
diff --git a/example/src/ecommerce_product_description.config.ts b/example/src/ecommerce_product_description.config.ts
@@ -0,0 +1,37 @@
+import 'dotenv/config'
+
+import { visionTool } from '@fabrice-ai/tools/vision'
+import { agent } from 'fabrice-ai/agent'
+import { workflow } from 'fabrice-ai/workflow'
+import path from 'path'
+
+const techExpert = agent({
+  description: `
+    You are skilled at extracting and describing most detailed technical information about the product from the photo.
+  `,
+  tools: {
+    visionTool,
+  },
+})
+
+const marketingManager = agent({
+  description: `
+    You are skilled at writing catchy product descriptions making customers to instantly fall in love with the product.
+    You always answer why they should buy the product, how it will make their life better, 
+    and what emotions it will evoke.
+  `,
+})
+
+export const productDescriptionWorkflow = workflow({
+  team: { techExpert, marketingManager },
+  description: `
+    Based on the picture of the product, make the product description to list it on the website.
+  `,
+  knowledge: `
+    Focus on all technical features of the product, including color, size, material, brand if possible, etc.
+    Picture is at "${path.resolve(import.meta.dirname, '../assets/example-sneakers.jpg')}". 
+  `,
+  output: `
+    Catchy product description covering all the product features.
+  `,
+})
diff --git a/example/src/ecommerce_product_description.test.ts b/example/src/ecommerce_product_description.test.ts
@@ -0,0 +1,30 @@
+import 'dotenv/config'
+
+import { suite, test } from '@fabrice-ai/bdd/suite'
+import { testwork } from '@fabrice-ai/bdd/testwork'
+
+import { productDescriptionWorkflow } from './ecommerce_product_description.config.js'
+
+const testResults = await testwork(
+  productDescriptionWorkflow,
+  suite({
+    description: 'Black box testing suite',
+    team: {
+      techExpert: [test('0_wikipedia', 'Should use "visionTool"')],
+    },
+    workflow: [
+      test(
+        '1_photo_description',
+        'The photo shows blue pair of shoes. Make sure the description includes the color and type of the shoes'
+      ),
+    ],
+  })
+)
+
+if (!testResults.passed) {
+  console.log('🚨 Test suite failed')
+  process.exit(-1)
+} else {
+  console.log('✅ Test suite passed')
+  process.exit(0)
+}
diff --git a/example/src/ecommerce_product_description.ts b/example/src/ecommerce_product_description.ts
@@ -1,42 +1,10 @@
 import 'dotenv/config'
 
-import { visionTool } from '@fabrice-ai/tools/vision'
-import { agent } from 'fabrice-ai/agent'
 import { solution } from 'fabrice-ai/solution'
 import { teamwork } from 'fabrice-ai/teamwork'
-import { workflow } from 'fabrice-ai/workflow'
-import path from 'path'
 
-const techExpert = agent({
-  description: `
-    You are skilled at extracting and describing most detailed technical information about the product from the photo.
-  `,
-  tools: {
-    visionTool,
-  },
-})
+import { productDescriptionWorkflow } from './ecommerce_product_description.config.js'
 
-const marketingManager = agent({
-  description: `
-    You are skilled at writing catchy product descriptions making customers to instantly fall in love with the product.
-    You always answer why they should buy the product, how it will make their life better, 
-    and what emotions it will evoke.
-  `,
-})
-
-const productDescriptionWorkflow = workflow({
-  team: { techExpert, marketingManager },
-  description: `
-    Based on the picture of the product, make the product description to list it on the website.
-  `,
-  knowledge: `
-    Focus on all technical features of the product, including color, size, material, brand if possible, etc.
-    Picture is at "${path.resolve(import.meta.dirname, '../assets/example-sneakers.jpg')}". 
-  `,
-  output: `
-    Catchy product description covering all the product features.
-  `,
-})
 const result = await teamwork(productDescriptionWorkflow)
 
 console.log(solution(result))
diff --git a/example/src/github_trending.ts b/example/src/github_trending.ts
diff --git a/example/src/github_trending_vector.config.ts b/example/src/github_trending_vector.config.ts
@@ -0,0 +1,69 @@
+import 'dotenv/config'
+
+import { createFireCrawlTool } from '@fabrice-ai/tools/firecrawl'
+import { getApiKey } from '@fabrice-ai/tools/utils'
+import { createVectorStoreTools } from '@fabrice-ai/tools/vector'
+import { agent } from 'fabrice-ai/agent'
+import { logger } from 'fabrice-ai/telemetry'
+import { workflow } from 'fabrice-ai/workflow'
+
+import { askUser } from './tools/askUser.js'
+
+const apiKey = await getApiKey('Firecrawl.dev API Key', 'FIRECRAWL_API_KEY')
+
+const { saveDocumentInVectorStore, searchInVectorStore } = createVectorStoreTools()
+
+const { firecrawl } = createFireCrawlTool({
+  apiKey,
+})
+
+const webCrawler = agent({
+  description: `
+    You are skilled at browsing Web pages.
+    You can save the documents to Vector store for later usage.
+  `,
+  tools: {
+    firecrawl,
+    saveDocumentInVectorStore,
+  },
+})
+
+const human = agent({
+  description: `
+    You can ask user and get their answer to questions that are needed by other agents.
+  `,
+  tools: {
+    askUser,
+  },
+})
+
+const reportCompiler = agent({
+  description: `
+    You can create a comprehensive report based on the information from Vector store.
+    You're famous for beautiful Markdown formatting.
+  `,
+  tools: {
+    searchInVectorStore,
+  },
+})
+
+export const wrapUpTrending = workflow({
+  team: { webCrawler, human, reportCompiler },
+  description: `
+    Research the "https://github.com/trending/typescript" page.
+    Select 3 top projects. 
+    For each project, browse details about it on their subpages.
+    Store each page in Vector store for later usage.
+  
+    Ask user about which project he wants to learn more.
+   `,
+  knowledge: `
+    Each document in Vector store is a page from the website.
+  `,
+  output: `
+    Create a comprehensive markdown report:
+     - create a one, two sentences summary about every project.
+     - include detailed summary about the project selected by the user. 
+  `,
+  snapshot: logger,
+})
diff --git a/example/src/github_trending_vector.test.ts b/example/src/github_trending_vector.test.ts
@@ -0,0 +1,34 @@
+import 'dotenv/config'
+
+import { suite, test } from '@fabrice-ai/bdd/suite'
+import { testwork } from '@fabrice-ai/bdd/testwork'
+
+import { wrapUpTrending } from './github_trending_vector.config.js'
+
+const testResults = await testwork(
+  wrapUpTrending,
+  suite({
+    description: 'Black box testing suite',
+    team: {
+      webCrawler: [
+        test(
+          '0_webCrawler',
+          'Should use "firecrawl" to crawl Github and may store data in the vector store using "saveDocumentInVectorStore"'
+        ),
+      ],
+    },
+    workflow: [
+      test('1_check_the_list', 'Should find 3 trending projects on Github'),
+      test('2_check_the_list', 'Should ask the user for one of these projects'),
+      test('3_details', 'Should generate the report with the details of the selected project'),
+    ],
+  })
+)
+
+if (!testResults.passed) {
+  console.log('🚨 Test suite failed')
+  process.exit(-1)
+} else {
+  console.log('✅ Test suite passed')
+  process.exit(0)
+}
diff --git a/example/src/github_trending_vector.ts b/example/src/github_trending_vector.ts
@@ -1,74 +1,9 @@
 import 'dotenv/config'
 
-import { createFireCrawlTool } from '@fabrice-ai/tools/firecrawl'
-import { getApiKey } from '@fabrice-ai/tools/utils'
-import { createVectorStoreTools } from '@fabrice-ai/tools/vector'
-import { agent } from 'fabrice-ai/agent'
 import { solution } from 'fabrice-ai/solution'
 import { teamwork } from 'fabrice-ai/teamwork'
-import { logger } from 'fabrice-ai/telemetry'
-import { workflow } from 'fabrice-ai/workflow'
 
-import { askUser } from './tools/askUser.js'
-
-const apiKey = await getApiKey('Firecrawl.dev API Key', 'FIRECRAWL_API_KEY')
-
-const { saveDocumentInVectorStore, searchInVectorStore } = createVectorStoreTools()
-
-const { firecrawl } = createFireCrawlTool({
-  apiKey,
-})
-
-const webCrawler = agent({
-  description: `
-    You are skilled at browsing Web pages.
-    You can save the documents to Vector store for later usage.
-  `,
-  tools: {
-    firecrawl,
-    saveDocumentInVectorStore,
-  },
-})
-
-const human = agent({
-  description: `
-    You can ask user and get their answer to questions that are needed by other agents.
-  `,
-  tools: {
-    askUser,
-  },
-})
-
-const reportCompiler = agent({
-  description: `
-    You can create a comprehensive report based on the information from Vector store.
-    You're famous for beautiful Markdown formatting.
-  `,
-  tools: {
-    searchInVectorStore,
-  },
-})
-
-const wrapUpTrending = workflow({
-  team: { webCrawler, human, reportCompiler },
-  description: `
-    Research the "https://github.com/trending/typescript" page.
-    Select 3 top projects. 
-    For each project, browse details about it on their subpages.
-    Store each page in Vector store for later usage.
-  
-    Ask user about which project he wants to learn more.
-   `,
-  knowledge: `
-    Each document in Vector store is a page from the website.
-  `,
-  output: `
-    Create a comprehensive markdown report:
-     - create a one, two sentences summary about every project.
-     - include detailed summary about the project selected by the user. 
-  `,
-  snapshot: logger,
-})
+import { wrapUpTrending } from './github_trending_vector.config.js'
 
 const result = await teamwork(wrapUpTrending)