Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add option to return full specs in GetList #191

Merged
merged 1 commit into from
Jan 28, 2025
Merged

add option to return full specs in GetList #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 37 additions & 7 deletions pkg/registry/file/sqlite.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,17 @@ func NewTestPool(dir string) *sqlitemigration.Pool {
return NewPool(filepath.Join(dir, "test.sq3"), 0)
}

func pathToKeys(path string) (string, string, string) {
func pathToKeys(path string) (string, string, string, string, string) {
s := strings.SplitN(path, "/", 5)
// ensure we have at least 5 parts
for len(s) < 5 {
s = append(s, "")
}
return s[2], s[3], s[4]
return s[0], s[1], s[2], s[3], s[4]
}

func countMetadata(conn *sqlite.Conn, path string) (int64, error) {
kind, namespace, _ := pathToKeys(path)
_, _, kind, namespace, _ := pathToKeys(path)
var count int64
err := sqlitex.Execute(conn,
`SELECT COUNT(*) FROM metadata
Expand All @@ -69,7 +69,7 @@ func countMetadata(conn *sqlite.Conn, path string) (int64, error) {
}

func DeleteMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) error {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
err := sqlitex.ExecuteTransient(conn,
`DELETE FROM metadata
WHERE kind = :kind
Expand All @@ -92,8 +92,38 @@ func DeleteMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) err
return nil
}

func listKeys(conn *sqlite.Conn, path, cont string, limit int64) ([]string, string, error) {
prefix, root, kind, namespace, _ := pathToKeys(path)
if cont == "" {
cont = "0"
}
var last string
var names []string
err := sqlitex.Execute(conn,
`SELECT rowid, namespace, name FROM metadata
WHERE kind = :kind
AND (:namespace = '' OR namespace = :namespace)
AND rowid > :cont
ORDER BY rowid
LIMIT :limit`,
&sqlitex.ExecOptions{
Named: map[string]any{":kind": kind, ":namespace": namespace, ":cont": cont, ":limit": limit},
ResultFunc: func(stmt *sqlite.Stmt) error {
last = stmt.ColumnText(0)
ns := stmt.ColumnText(1)
name := stmt.ColumnText(2)
names = append(names, fmt.Sprintf("%s/%s/%s/%s/%s", prefix, root, kind, ns, name))
return nil
},
})
if err != nil {
return nil, "", fmt.Errorf("list names: %w", err)
}
return names, last, nil
}

func listMetadata(conn *sqlite.Conn, path, cont string, limit int64) ([]string, string, error) {
kind, namespace, _ := pathToKeys(path)
_, _, kind, namespace, _ := pathToKeys(path)
if cont == "" {
cont = "0"
}
Expand Down Expand Up @@ -122,7 +152,7 @@ func listMetadata(conn *sqlite.Conn, path, cont string, limit int64) ([]string,
}

func ReadMetadata(conn *sqlite.Conn, path string) ([]byte, error) {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
var metadataJSON string
err := sqlitex.Execute(conn,
`SELECT metadata FROM metadata
Expand Down Expand Up @@ -154,7 +184,7 @@ func writeMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) erro
}

func WriteJSON(conn *sqlite.Conn, path string, metadataJSON []byte) error {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
err := sqlitex.ExecuteTransient(conn,
`INSERT OR REPLACE INTO metadata
(kind, namespace, name, metadata) VALUES (?, ?, ?, ?)`,
Expand Down
2 changes: 1 addition & 1 deletion pkg/registry/file/sqlite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func Test_pathToKindKey(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.test, func(t *testing.T) {
kind, namespace, name := pathToKeys(tt.path)
_, _, kind, namespace, name := pathToKeys(tt.path)
assert.Equalf(t, tt.kind, kind, "pathToKeys(%v)", tt.path)
assert.Equalf(t, tt.namespace, namespace, "pathToKeys(%v)", tt.path)
assert.Equalf(t, tt.name, name, "pathToKeys(%v)", tt.path)
Expand Down
46 changes: 33 additions & 13 deletions pkg/registry/file/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,27 +392,47 @@ func (s *StorageImpl) GetList(ctx context.Context, key string, opts storage.List
if opts.Predicate.Limit == 0 {
opts.Predicate.Limit = 500
}
// get metadata from SQLite
// prepare SQLite connection
conn, err := s.pool.Take(context.Background())
if err != nil {
return fmt.Errorf("take connection: %w", err)
}
defer s.pool.Put(conn)
metadataJSONs, last, err := listMetadata(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list metadata failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, metadataJSON := range metadataJSONs {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := json.Unmarshal([]byte(metadataJSON), obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - unmarshal metadata failed", helpers.Error(err), helpers.String("key", key))
var list []string
var last string
if opts.ResourceVersion == "fullSpec" {
// get names from SQLite
list, last, err = listKeys(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list keys failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, k := range list {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := s.get(ctx, k, storage.GetOptions{}, obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - get object failed", helpers.Error(err), helpers.String("key", k))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
} else {
// get metadata from SQLite
list, last, err = listMetadata(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list metadata failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, metadataJSON := range list {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := json.Unmarshal([]byte(metadataJSON), obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - unmarshal metadata failed", helpers.Error(err), helpers.String("key", key))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
// eventually set list accessor fields
if len(metadataJSONs) == int(opts.Predicate.Limit) {
if len(list) == int(opts.Predicate.Limit) {
listAccessor, err := meta.ListAccessor(listObj)
if err != nil {
return fmt.Errorf("list accessor: %w", err)
Expand Down
Loading