Skip to content

Commit

Permalink
Removed 'goal'
Browse files Browse the repository at this point in the history
  • Loading branch information
@teilomillet
teilomillet committed Nov 21, 2024
1 parent c37575b commit 6872d54
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 61 deletions.
26 changes: 13 additions & 13 deletions examples/full_process.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"sync"
"time"

"github.com/teilomillet/goal"
"github.com/teilomillet/gollm"
"github.com/teilomillet/raggo"
"golang.org/x/time/rate"
)
Expand Down Expand Up @@ -87,19 +87,19 @@ func main() {
}

embedder, err := raggo.NewEmbedder(
raggo.SetProvider("openai"),
raggo.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.SetModel("text-embedding-3-small"),
raggo.WithEmbedderProvider("openai"),
raggo.WithEmbedderAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.WithEmbedderModel("text-embedding-3-small"),
)
if err != nil {
log.Fatalf("Failed to create embedder: %v", err)
}

llm, err := goal.NewLLM(
goal.SetProvider("openai"),
goal.SetModel("gpt-4o-mini"),
goal.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
goal.SetMaxTokens(2048),
llm, err := gollm.NewLLM(
gollm.SetProvider("openai"),
gollm.SetModel("gpt-4o-mini"),
gollm.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
gollm.SetMaxTokens(2048),
)
if err != nil {
log.Fatalf("Failed to create LLM: %v", err)
Expand Down Expand Up @@ -177,7 +177,7 @@ func createMilvusCollection(milvusDB *raggo.VectorDB, collectionName string) {
}
}

func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm goal.LLM, milvusDB *raggo.VectorDB, collectionName, targetDir string) {
func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm gollm.LLM, milvusDB *raggo.VectorDB, collectionName, targetDir string) {
files, err := filepath.Glob(filepath.Join(targetDir, "*.pdf"))
if err != nil {
log.Fatalf("Failed to list PDF files: %v", err)
Expand Down Expand Up @@ -243,7 +243,7 @@ func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder
fmt.Printf("Average summaries per second: %.2f\n", float64(summaryCount)/duration.Seconds())
}

func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm goal.LLM, milvusDB *raggo.VectorDB, collectionName, filePath string) (int, int, int, error) {
func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm gollm.LLM, milvusDB *raggo.VectorDB, collectionName, filePath string) (int, int, int, error) {
log.Printf("Processing file: %s", filePath)

doc, err := parser.Parse(filePath)
Expand All @@ -260,14 +260,14 @@ func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder rag
return 0, 0, 0, fmt.Errorf("rate limit wait error: %w", err)
}

summaryPrompt := goal.NewPrompt(fmt.Sprintf("Summarize the following text in 2-3 sentences:\n\n%s", doc.Content))
summaryPrompt := gollm.NewPrompt(fmt.Sprintf("Summarize the following text in 2-3 sentences:\n\n%s", doc.Content))
summary, err := llm.Generate(context.Background(), summaryPrompt)
if err != nil {
return 0, 0, 0, fmt.Errorf("error generating summary: %w", err)
}

// Update rate limiter based on response headers
// Note: You'll need to modify your goal.LLM interface to return these headers
// Note: You'll need to modify your gollm.LLM interface to return these headers
// gptLimiter.UpdateLimits(remainingRequests, remainingTokens, resetRequests, resetTokens)

chunks := chunker.Chunk(summary)
Expand Down
24 changes: 12 additions & 12 deletions examples/process_embedding_benchmark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"sync"
"time"

"github.com/teilomillet/goal"
"github.com/teilomillet/gollm"
"github.com/teilomillet/raggo"
)

Expand All @@ -35,19 +35,19 @@ func main() {
}

embedder, err := raggo.NewEmbedder(
raggo.SetProvider("openai"),
raggo.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.SetModel("text-embedding-3-small"),
raggo.WithEmbedderProvider("openai"),
raggo.WithEmbedderAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.WithEmbedderModel("text-embedding-3-small"),
)
if err != nil {
log.Fatalf("Failed to create embedder: %v", err)
}

llm, err := goal.NewLLM(
goal.SetProvider("openai"),
goal.SetModel("gpt-4o-mini"),
goal.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
goal.SetMaxTokens(2048),
llm, err := gollm.NewLLM(
gollm.SetProvider("openai"),
gollm.SetModel("gpt-4o-mini"),
gollm.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
gollm.SetMaxTokens(2048),
)
if err != nil {
log.Fatalf("Failed to create LLM: %v", err)
Expand All @@ -56,7 +56,7 @@ func main() {
benchmarkPDFProcessing(parser, chunker, embedder, llm, targetDir)
}

func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm goal.LLM, targetDir string) {
func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm gollm.LLM, targetDir string) {
files, err := filepath.Glob(filepath.Join(targetDir, "*.pdf"))
if err != nil {
log.Fatalf("Failed to list PDF files: %v", err)
Expand Down Expand Up @@ -116,7 +116,7 @@ func benchmarkPDFProcessing(parser raggo.Parser, chunker raggo.Chunker, embedder
fmt.Printf("Average summaries per second: %.2f\n", float64(summaryCount)/duration.Seconds())
}

func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm goal.LLM, filePath string) (int, int, int, error) {
func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder raggo.Embedder, llm gollm.LLM, filePath string) (int, int, int, error) {
log.Printf("Processing file: %s", filePath)

doc, err := parser.Parse(filePath)
Expand All @@ -130,7 +130,7 @@ func processAndEmbedPDF(parser raggo.Parser, chunker raggo.Chunker, embedder rag

log.Printf("Successfully parsed PDF: %s, Content length: %d", filePath, len(doc.Content))

summaryPrompt := goal.NewPrompt(fmt.Sprintf("Summarize the following text in 2-3 sentences:\n\n%s", doc.Content))
summaryPrompt := gollm.NewPrompt(fmt.Sprintf("Summarize the following text in 2-3 sentences:\n\n%s", doc.Content))
summary, err := llm.Generate(context.Background(), summaryPrompt)
if err != nil {
return 0, 0, 0, fmt.Errorf("error generating summary: %w", err)
Expand Down
73 changes: 37 additions & 36 deletions examples/recruit_example.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"sync"
"time"

"github.com/teilomillet/goal"
"github.com/teilomillet/gollm"
"github.com/teilomillet/raggo"
)

Expand Down Expand Up @@ -49,11 +49,11 @@ func main() {

// Initialize the LLM
debug("Initializing LLM...")
llm, err := goal.NewLLM(
goal.SetProvider("openai"),
goal.SetModel("gpt-4o-mini"),
goal.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
goal.SetMaxTokens(2048),
llm, err := gollm.NewLLM(
gollm.SetProvider("openai"),
gollm.SetModel("gpt-4o-mini"),
gollm.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
gollm.SetMaxTokens(2048),
)
if err != nil {
log.Fatalf("Failed to create LLM: %v", err)
Expand All @@ -77,9 +77,9 @@ func main() {

debug("Initializing embedder...")
embedder, err := raggo.NewEmbedder(
raggo.SetProvider("openai"),
raggo.SetAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.SetModel("text-embedding-3-small"),
raggo.WithEmbedderProvider("openai"),
raggo.WithEmbedderAPIKey(os.Getenv("OPENAI_API_KEY")),
raggo.WithEmbedderModel("text-embedding-3-small"),
)
if err != nil {
log.Fatalf("Failed to create embedder: %v", err)
Expand Down Expand Up @@ -212,7 +212,7 @@ func main() {
printCandidates(hybridCandidates)
}

func processResumes(llm goal.LLM, parser raggo.Parser, chunker raggo.Chunker, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, resumeDir string) error {
func processResumes(llm gollm.LLM, parser raggo.Parser, chunker raggo.Chunker, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, resumeDir string) error {
fmt.Printf("Scanning directory: %s\n", resumeDir)
files, err := filepath.Glob(filepath.Join(resumeDir, "*.pdf"))
if err != nil {
Expand Down Expand Up @@ -252,7 +252,7 @@ func processResumes(llm goal.LLM, parser raggo.Parser, chunker raggo.Chunker, em
return nil
}

func processResume(file string, llm goal.LLM, parser raggo.Parser, chunker raggo.Chunker, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, cacheDir string) error {
func processResume(file string, llm gollm.LLM, parser raggo.Parser, chunker raggo.Chunker, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, cacheDir string) error {
log.Printf("Processing resume: %s", file)

cacheFile := filepath.Join(cacheDir, filepath.Base(file)+".cache")
Expand Down Expand Up @@ -371,8 +371,8 @@ func deduplicateResults(results []raggo.SearchResult) []raggo.SearchResult {
return deduplicated
}

func createStandardizedSummary(llm goal.LLM, content string) (string, error) {
summarizePrompt := goal.NewPrompt(
func createStandardizedSummary(llm gollm.LLM, content string) (string, error) {
summarizePrompt := gollm.NewPrompt(
"Summarize the given resume in the following standardized format:\n" +
"Personal Information: [Name, contact details]\n" +
"Professional Summary: [2-3 sentences summarizing key qualifications and career objectives]\n" +
Expand All @@ -392,24 +392,25 @@ func createStandardizedSummary(llm goal.LLM, content string) (string, error) {
return standardizedSummary, nil
}

func extractStructuredData(llm goal.LLM, standardizedSummary string) (*ResumeInfo, error) {
extractPrompt := goal.NewPrompt(fmt.Sprintf(`
Extract the following information from the given resume summary:
- Name
- Contact Details
- Professional Summary
- Skills (as a list)
- Work Experience
- Education
- Additional Information
Resume Summary:
%s
Respond with a JSON object containing these fields.
`, standardizedSummary))

resumeInfo, err := goal.ExtractStructuredData[ResumeInfo](context.Background(), llm, extractPrompt.String())
func extractStructuredData(llm gollm.LLM, standardizedSummary string) (*ResumeInfo, error) {
extractPrompt := gollm.NewPrompt(fmt.Sprintf(`
Extract the following information from the given resume summary:
- Name
- Contact Details
- Professional Summary
- Skills (as a list)
- Work Experience
- Education
- Additional Information
Resume Summary:
%s
Respond with a JSON object containing these fields.
`, standardizedSummary))

resumeInfo := &ResumeInfo{}
_, err := llm.GenerateWithSchema(context.Background(), extractPrompt, resumeInfo)
if err != nil {
return nil, fmt.Errorf("failed to extract structured data: %w", err)
}
Expand Down Expand Up @@ -444,12 +445,12 @@ func getCollectionItemCount(vectorDB raggo.VectorDB, collectionName string) (int
return 0, fmt.Errorf("getCollectionItemCount not implemented")
}

func searchCandidates(ctx context.Context, llm goal.LLM, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, jobOffer string) ([]raggo.SearchResult, error) {
func searchCandidates(ctx context.Context, llm gollm.LLM, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, jobOffer string) ([]raggo.SearchResult, error) {

fmt.Println("Starting candidate search...")

// Summarize the job offer
jobSummary, err := goal.Summarize(ctx, llm, jobOffer)
jobSummary, err := llm.Generate(ctx, llm.NewPrompt(jobOffer))
if err != nil {
return nil, fmt.Errorf("failed to summarize job offer: %w", err)
}
Expand Down Expand Up @@ -489,7 +490,7 @@ func searchCandidates(ctx context.Context, llm goal.LLM, embeddingService *raggo
return results, nil
}

func hybridSearchCandidates(ctx context.Context, llm goal.LLM, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, jobOffer string) ([]raggo.SearchResult, error) {
func hybridSearchCandidates(ctx context.Context, llm gollm.LLM, embeddingService *raggo.EmbeddingService, vectorDB *raggo.VectorDB, jobOffer string) ([]raggo.SearchResult, error) {
fmt.Println("Starting optimized hybrid search for candidates...")

// Extract different aspects of the job offer
Expand Down Expand Up @@ -564,8 +565,8 @@ func hybridSearchCandidates(ctx context.Context, llm goal.LLM, embeddingService
return dedupedResults, nil
}

func extractJobAspects(ctx context.Context, llm goal.LLM, jobOffer string) (map[string]string, error) {
prompt := goal.NewPrompt(`
func extractJobAspects(ctx context.Context, llm gollm.LLM, jobOffer string) (map[string]string, error) {
prompt := gollm.NewPrompt(`
Extract and summarize the following aspects from the job offer:
1. Required Skills
2. Experience Level
Expand Down

0 comments on commit 6872d54

Please sign in to comment.