Skip to content

Commit

Permalink
Merge pull request #191 from kubescape/fullspec
Browse files Browse the repository at this point in the history 
add option to return full specs in GetList
  • Loading branch information
@matthyx
matthyx authored Jan 28, 2025
2 parents 4aaba20 + 557b26e commit 7d31d1a
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 21 deletions.
44 changes: 37 additions & 7 deletions pkg/registry/file/sqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,17 @@ func NewTestPool(dir string) *sqlitemigration.Pool {
return NewPool(filepath.Join(dir, "test.sq3"), 0)
}

func pathToKeys(path string) (string, string, string) {
func pathToKeys(path string) (string, string, string, string, string) {
s := strings.SplitN(path, "/", 5)
// ensure we have at least 5 parts
for len(s) < 5 {
s = append(s, "")
}
return s[2], s[3], s[4]
return s[0], s[1], s[2], s[3], s[4]
}

func countMetadata(conn *sqlite.Conn, path string) (int64, error) {
kind, namespace, _ := pathToKeys(path)
_, _, kind, namespace, _ := pathToKeys(path)
var count int64
err := sqlitex.Execute(conn,
`SELECT COUNT(*) FROM metadata
Expand All @@ -69,7 +69,7 @@ func countMetadata(conn *sqlite.Conn, path string) (int64, error) {
}

func DeleteMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) error {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
err := sqlitex.ExecuteTransient(conn,
`DELETE FROM metadata
WHERE kind = :kind
Expand All @@ -92,8 +92,38 @@ func DeleteMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) err
return nil
}

func listKeys(conn *sqlite.Conn, path, cont string, limit int64) ([]string, string, error) {
prefix, root, kind, namespace, _ := pathToKeys(path)
if cont == "" {
cont = "0"
}
var last string
var names []string
err := sqlitex.Execute(conn,
`SELECT rowid, namespace, name FROM metadata
WHERE kind = :kind
AND (:namespace = '' OR namespace = :namespace)
AND rowid > :cont
ORDER BY rowid
LIMIT :limit`,
&sqlitex.ExecOptions{
Named: map[string]any{":kind": kind, ":namespace": namespace, ":cont": cont, ":limit": limit},
ResultFunc: func(stmt *sqlite.Stmt) error {
last = stmt.ColumnText(0)
ns := stmt.ColumnText(1)
name := stmt.ColumnText(2)
names = append(names, fmt.Sprintf("%s/%s/%s/%s/%s", prefix, root, kind, ns, name))
return nil
},
})
if err != nil {
return nil, "", fmt.Errorf("list names: %w", err)
}
return names, last, nil
}

func listMetadata(conn *sqlite.Conn, path, cont string, limit int64) ([]string, string, error) {
kind, namespace, _ := pathToKeys(path)
_, _, kind, namespace, _ := pathToKeys(path)
if cont == "" {
cont = "0"
}
Expand Down Expand Up @@ -122,7 +152,7 @@ func listMetadata(conn *sqlite.Conn, path, cont string, limit int64) ([]string,
}

func ReadMetadata(conn *sqlite.Conn, path string) ([]byte, error) {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
var metadataJSON string
err := sqlitex.Execute(conn,
`SELECT metadata FROM metadata
Expand Down Expand Up @@ -154,7 +184,7 @@ func writeMetadata(conn *sqlite.Conn, path string, metadata runtime.Object) erro
}

func WriteJSON(conn *sqlite.Conn, path string, metadataJSON []byte) error {
kind, namespace, name := pathToKeys(path)
_, _, kind, namespace, name := pathToKeys(path)
err := sqlitex.ExecuteTransient(conn,
`INSERT OR REPLACE INTO metadata
(kind, namespace, name, metadata) VALUES (?, ?, ?, ?)`,
Expand Down
2 changes: 1 addition & 1 deletion pkg/registry/file/sqlite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func Test_pathToKindKey(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.test, func(t *testing.T) {
kind, namespace, name := pathToKeys(tt.path)
_, _, kind, namespace, name := pathToKeys(tt.path)
assert.Equalf(t, tt.kind, kind, "pathToKeys(%v)", tt.path)
assert.Equalf(t, tt.namespace, namespace, "pathToKeys(%v)", tt.path)
assert.Equalf(t, tt.name, name, "pathToKeys(%v)", tt.path)
Expand Down
46 changes: 33 additions & 13 deletions pkg/registry/file/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,27 +392,47 @@ func (s *StorageImpl) GetList(ctx context.Context, key string, opts storage.List
if opts.Predicate.Limit == 0 {
opts.Predicate.Limit = 500
}
// get metadata from SQLite
// prepare SQLite connection
conn, err := s.pool.Take(context.Background())
if err != nil {
return fmt.Errorf("take connection: %w", err)
}
defer s.pool.Put(conn)
metadataJSONs, last, err := listMetadata(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list metadata failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, metadataJSON := range metadataJSONs {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := json.Unmarshal([]byte(metadataJSON), obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - unmarshal metadata failed", helpers.Error(err), helpers.String("key", key))
var list []string
var last string
if opts.ResourceVersion == "fullSpec" {
// get names from SQLite
list, last, err = listKeys(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list keys failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, k := range list {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := s.get(ctx, k, storage.GetOptions{}, obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - get object failed", helpers.Error(err), helpers.String("key", k))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
} else {
// get metadata from SQLite
list, last, err = listMetadata(conn, key, opts.Predicate.Continue, opts.Predicate.Limit)
if err != nil {
logger.L().Ctx(ctx).Error("GetList - list metadata failed", helpers.Error(err), helpers.String("key", key))
}
// populate list object
for _, metadataJSON := range list {
elem := v.Type().Elem()
obj := reflect.New(elem).Interface().(runtime.Object)
if err := json.Unmarshal([]byte(metadataJSON), obj); err != nil {
logger.L().Ctx(ctx).Error("GetList - unmarshal metadata failed", helpers.Error(err), helpers.String("key", key))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
}
// eventually set list accessor fields
if len(metadataJSONs) == int(opts.Predicate.Limit) {
if len(list) == int(opts.Predicate.Limit) {
listAccessor, err := meta.ListAccessor(listObj)
if err != nil {
return fmt.Errorf("list accessor: %w", err)
Expand Down

0 comments on commit 7d31d1a

Please sign in to comment.