-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: knowledge - re-use embeddings on document layer (copy embedding…
…s from docs with same content) (#444)
- Loading branch information
1 parent
2a5e7ee
commit 73689ed
Showing
19 changed files
with
315 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package helper | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| cg "github.com/philippgille/chromem-go" | ||
| ) | ||
|
|
||
| func BuildWhereDocumentClause(whereDocs []cg.WhereDocument, joinOperator string) (string, error) { | ||
| if len(whereDocs) == 0 { | ||
| return "TRUE", nil | ||
| } | ||
| if joinOperator == "" { | ||
| joinOperator = "AND" | ||
| } | ||
| joinOperator = fmt.Sprintf(" %s ", strings.TrimSpace(joinOperator)) // ensure space around operator | ||
| var whereClauses []string | ||
| for _, wd := range whereDocs { | ||
| switch wd.Operator { | ||
| case cg.WhereDocumentOperatorAnd: | ||
| wc, err := BuildWhereDocumentClause(wd.WhereDocuments, "AND") | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| whereClauses = append(whereClauses, fmt.Sprintf("(%s)", wc)) | ||
| case cg.WhereDocumentOperatorOr: | ||
| wc, err := BuildWhereDocumentClause(wd.WhereDocuments, "OR") | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| whereClauses = append(whereClauses, fmt.Sprintf("(%s)", wc)) | ||
| case cg.WhereDocumentOperatorEquals: | ||
| whereClauses = append(whereClauses, fmt.Sprintf("document = '%s'", wd.Value)) | ||
| case cg.WhereDocumentOperatorContains: | ||
| whereClauses = append(whereClauses, fmt.Sprintf("document LIKE '%%%s%%'", wd.Value)) | ||
| case cg.WhereDocumentOperatorNotContains: | ||
| whereClauses = append(whereClauses, fmt.Sprintf("document NOT LIKE '%%%s%%'", wd.Value)) | ||
| } | ||
| } | ||
| return strings.Join(whereClauses, joinOperator), nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| package helper | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| cg "github.com/philippgille/chromem-go" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestBuildWhereDocumentClause_EmptyInput_TRUEClause(t *testing.T) { | ||
| var whereDocs []cg.WhereDocument | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "TRUE", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_SingleEqualsCondition_ReturnsCorrectClause(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test"}, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "document = 'test'", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_SingleContainsCondition_ReturnsCorrectClause(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorContains, Value: "test"}, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "document LIKE '%test%'", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_SingleNotContainsCondition_ReturnsCorrectClause(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorNotContains, Value: "test"}, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "document NOT LIKE '%test%'", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_AndCondition_ReturnsCorrectClauses(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| { | ||
| Operator: cg.WhereDocumentOperatorAnd, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test1"}, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test2"}, | ||
| }, | ||
| }, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "(document = 'test1' AND document = 'test2')", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_OrCondition_ReturnsCorrectClauses(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| { | ||
| Operator: cg.WhereDocumentOperatorOr, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test1"}, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test2"}, | ||
| }, | ||
| }, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "OR") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "(document = 'test1' OR document = 'test2')", whereClause) | ||
| } | ||
|
|
||
| func TestBuildWhereDocumentClause_Nested_ReturnsCorrectClauses(t *testing.T) { | ||
| whereDocs := []cg.WhereDocument{ | ||
| { | ||
| Operator: cg.WhereDocumentOperatorOr, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test1"}, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test2"}, | ||
| }, | ||
| }, | ||
| { | ||
| Operator: cg.WhereDocumentOperatorAnd, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test3"}, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test4"}, | ||
| }, | ||
| }, | ||
| { | ||
| Operator: cg.WhereDocumentOperatorAnd, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| { | ||
| Operator: cg.WhereDocumentOperatorAnd, | ||
| WhereDocuments: []cg.WhereDocument{ | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test5"}, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test6"}, | ||
| }, | ||
| }, | ||
| {Operator: cg.WhereDocumentOperatorEquals, Value: "test7"}, | ||
| }, | ||
| }, | ||
| } | ||
| whereClause, err := BuildWhereDocumentClause(whereDocs, "AND") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, "(document = 'test1' OR document = 'test2') AND (document = 'test3' AND document = 'test4') AND ((document = 'test5' AND document = 'test6') AND document = 'test7')", whereClause) | ||
| } |
Oops, something went wrong.