forked from wishonia/wishonia
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
web and video search tools including Tavily, Exa, and SearXNG integra…
…tions for web search, and YouTube video search. Also, implemented PDF processing for converting PDF contents into structured WordPress posts while preserving hyperlinks. Took 47 seconds
- Loading branch information
Showing
18 changed files
with
1,779 additions
and
253 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| export * from './task-manager' | ||
| export * from './inquire' | ||
| export * from './query-suggestor' | ||
| export * from './researcher' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import { Copilot } from '@/app/search/components/copilot' | ||
| import { createStreamableUI, createStreamableValue } from 'ai/rsc' | ||
| import { CoreMessage, streamObject } from 'ai' | ||
| import { PartialInquiry, inquirySchema } from '@/lib/schema/inquiry' | ||
| import { getModel } from '../utils/index' | ||
|
|
||
| export async function inquire( | ||
| uiStream: ReturnType<typeof createStreamableUI>, | ||
| messages: CoreMessage[] | ||
| ) { | ||
| const objectStream = createStreamableValue<PartialInquiry>() | ||
| uiStream.update(<Copilot inquiry={objectStream.value} />) | ||
|
|
||
| let finalInquiry: PartialInquiry = {} | ||
| await streamObject({ | ||
| model: getModel(), | ||
| system: `As a professional web researcher, your role is to deepen your understanding of the user's input by conducting further inquiries when necessary. | ||
| After receiving an initial response from the user, carefully assess whether additional questions are absolutely essential to provide a comprehensive and accurate answer. Only proceed with further inquiries if the available information is insufficient or ambiguous. | ||
| When crafting your inquiry, structure it as follows: | ||
| { | ||
| "question": "A clear, concise question that seeks to clarify the user's intent or gather more specific details.", | ||
| "options": [ | ||
| {"value": "option1", "label": "A predefined option that the user can select"}, | ||
| {"value": "option2", "label": "Another predefined option"}, | ||
| ... | ||
| ], | ||
| "allowsInput": true/false, // Indicates whether the user can provide a free-form input | ||
| "inputLabel": "A label for the free-form input field, if allowed", | ||
| "inputPlaceholder": "A placeholder text to guide the user's free-form input" | ||
| } | ||
| Important: The "value" field in the options must always be in English, regardless of the user's language. | ||
| For example: | ||
| { | ||
| "question": "What specific information are you seeking about Rivian?", | ||
| "options": [ | ||
| {"value": "history", "label": "History"}, | ||
| {"value": "products", "label": "Products"}, | ||
| {"value": "investors", "label": "Investors"}, | ||
| {"value": "partnerships", "label": "Partnerships"}, | ||
| {"value": "competitors", "label": "Competitors"} | ||
| ], | ||
| "allowsInput": true, | ||
| "inputLabel": "If other, please specify", | ||
| "inputPlaceholder": "e.g., Specifications" | ||
| } | ||
| By providing predefined options, you guide the user towards the most relevant aspects of their query, while the free-form input allows them to provide additional context or specific details not covered by the options. | ||
| Remember, your goal is to gather the necessary information to deliver a thorough and accurate response. | ||
| Please match the language of the response (question, labels, inputLabel, and inputPlaceholder) to the user's language, but keep the "value" field in English. | ||
| `, | ||
| messages, | ||
| schema: inquirySchema | ||
| }) | ||
| .then(async result => { | ||
| for await (const obj of result.partialObjectStream) { | ||
| if (obj) { | ||
| objectStream.update(obj) | ||
| finalInquiry = obj | ||
| } | ||
| } | ||
| }) | ||
| .finally(() => { | ||
| objectStream.done() | ||
| }) | ||
|
|
||
| return finalInquiry | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import fetch from 'node-fetch'; | ||
| import fs from 'fs/promises'; | ||
| import path from 'path'; | ||
|
|
||
| export class PDFDownloader { | ||
| private downloadDir: string; | ||
|
|
||
| constructor(downloadDir: string) { | ||
| this.downloadDir = downloadDir; | ||
| } | ||
|
|
||
| private async ensureDirectoryExists(): Promise<void> { | ||
| try { | ||
| await fs.access(this.downloadDir); | ||
| } catch { | ||
| await fs.mkdir(this.downloadDir, { recursive: true }); | ||
| } | ||
| } | ||
|
|
||
| private async downloadPDF(pdfUrl: string): Promise<void> { | ||
| const response = await fetch(pdfUrl); | ||
| if (!response.ok) { | ||
| throw new Error(`Failed to download PDF: ${response.statusText}`); | ||
| } | ||
|
|
||
| const arrayBuffer = await response.arrayBuffer(); | ||
| const buffer = Buffer.from(arrayBuffer); | ||
|
|
||
| // Extract filename from URL or use a fallback | ||
| const urlParts = new URL(pdfUrl); | ||
| const fileName = path.basename(urlParts.pathname) || 'document.pdf'; | ||
| const filePath = path.join(this.downloadDir, fileName); | ||
|
|
||
| await fs.writeFile(filePath, buffer); | ||
| console.log(`Downloaded: ${fileName}`); | ||
| } | ||
|
|
||
| public async downloadPDFs(pdfUrls: string[]): Promise<{ | ||
| downloaded: number; | ||
| errors: string[]; | ||
| }> { | ||
| const results = { | ||
| downloaded: 0, | ||
| errors: [] as string[] | ||
| }; | ||
|
|
||
| try { | ||
| await this.ensureDirectoryExists(); | ||
|
|
||
| console.log(`Starting download of ${pdfUrls.length} PDFs...`); | ||
|
|
||
| for (const pdfUrl of pdfUrls) { | ||
| try { | ||
| await this.downloadPDF(pdfUrl); | ||
| results.downloaded++; | ||
| } catch (error) { | ||
| const errorMessage = error instanceof Error ? error.message : 'Unknown error'; | ||
| results.errors.push(`Error downloading ${pdfUrl}: ${errorMessage}`); | ||
| console.error(`Failed to download ${pdfUrl}: ${errorMessage}`); | ||
| } | ||
| } | ||
| } catch (error) { | ||
| const errorMessage = error instanceof Error ? error.message : 'Unknown error'; | ||
| results.errors.push(`Error in download process: ${errorMessage}`); | ||
| console.error(`Error in download process: ${errorMessage}`); | ||
| } | ||
|
|
||
| return results; | ||
| } | ||
| } |
Oops, something went wrong.