From 0fadc233f98ae9bf913a5c3526956bd45448fc33 Mon Sep 17 00:00:00 2001 From: MD Islam Date: Mon, 18 Mar 2024 12:56:49 -0400 Subject: [PATCH 1/2] feat(created_at_sort): add support for sorting by created Signed-off-by: MD Islam --- .gitignore | 1 + README.md | 17 +++++++-- check_test.go | 93 ++++++++++++++++++++++++++++++++++++++++++++--- commands/check.go | 24 ++++++++++++ types.go | 3 +- 5 files changed, 128 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index a804790..1ff899b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ testdata/.push-output .envrc tags +.idea diff --git a/README.md b/README.md index 1966289..1623848 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ to, and `out` will always push to the specified tag. This is to be used in simpler cases where no real versioning exists. With `tag_regex` specified, `check` will instead detect tags based on the regex -provided +provided. If `created_at_sort` is set to `true`, the tags will be sorted in descending order by the creation time. +This is useful when you want to get the latest tag based on the regex (see Docker registry issue +[here](https://github.com/docker/hub-feedback/issues/185)). With `tag` and `tag_regex` both omitted, `check` will instead detect tags based on semver versions (e.g. `1.2.3`) and return them in semver order. With `variant` included, @@ -89,9 +91,17 @@ differences:
The syntax of the regular expressions accepted is the same general syntax used by Perl, Python, and other languages. More precisely, it is the syntax accepted by RE2 and described at https://golang.org/s/re2syntax -
Note if used, this will override all Semver constraints and features +
Note if used, this will override all Semver constraints and features. + By default, order of tags is not guaranteed. If you want to sort the tags in descending order, set `created_at_sort` to `true`. + + created_at_sort (Optional)
Default: false + + If set to `true`, the tags will be sorted in descending order using the creation time from the image history. + This is useful when you want to get the latest tag based on the tag_regex. + + variant (Optional) @@ -352,7 +362,8 @@ Reports the current digest that the registry has for the tag configured in ### `check` Step (`check` script) with `tag_regex`: discover tags matching regex Reports the current digest that the registry has for tags matching the regex -configured in `source`. They will be returned in the same order that the source repository lists them. +configured in `source`. They will be returned in the same order that the source repository lists them unless `created_at_sort` +is set to `true`. ### `check` Step (`check` script) without `tag` or `tag_regex`: discover semver tags diff --git a/check_test.go b/check_test.go index 92102bb..6eb598e 100644 --- a/check_test.go +++ b/check_test.go @@ -2,12 +2,15 @@ package resource_test import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "encoding/pem" "fmt" "net/http" "os/exec" "strconv" + "time" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -806,8 +809,43 @@ var _ = DescribeTable("tracking semver tags", ImageName: "random-4", }, }, - Regex: "gr(a|e)y", - Versions: []string{"gray", "grey"}, + Regex: "gr(a|e)y", + CreatedAtSort: false, + Versions: []string{"gray", "grey"}, + }, + ), + Entry("simple tag regex where sorted is true", + SemverOrRegexTagCheckExample{ + Tags: []testTag{ + { + Tag: "1.0.0", + ImageName: "random-1", + }, + { + Tag: "non-semver-tag", + ImageName: "random-2", + }, + { + Tag: "gem-1338-git-4bd8a5e1a244", + ImageName: "random-3", + }, + { + Tag: "gem-181-git-6bd8a5e1a2b3", + ImageName: "random-4", + }, + { + Tag: "gem-1337-git-4bd8a5e1a244", + ImageName: "random-5", + }, + }, + TagsToTime: map[string]time.Time{ + "gem-1338-git-4bd8a5e1a244": time.Date(2024, 1, 4, 5, 0, 0, 0, time.UTC), + "gem-181-git-6bd8a5e1a2b3": time.Date(2024, 1, 4, 0, 0, 0, 0, time.UTC), + "gem-1337-git-4bd8a5e1a244": time.Date(2024, 1, 4, 4, 0, 0, 0, time.UTC), + }, + Regex: "gem-(\\d+)-git-([a-f0-9]{12})", + CreatedAtSort: true, + Versions: []string{"gem-182-git-6bd8a5e1a2b3", "gem-1337-git-4bd8a5e1a244", "gem-1338-git-4bd8a5e1a244"}, }, ), Entry("regex override semver constraint", @@ -888,8 +926,9 @@ var _ = DescribeTable("tracking semver tags", ImageName: "random-5", }, }, - Regex: "^[0-9a-f]{7}-dev$", - Versions: []string{"3bd8a5e-dev", "67e3c33-dev"}, + Regex: "^[0-9a-f]{7}-dev$", + CreatedAtSort: false, + Versions: []string{"3bd8a5e-dev", "67e3c33-dev"}, }, ), Entry("semver tag ordering", @@ -1488,12 +1527,14 @@ type testTag struct { } type SemverOrRegexTagCheckExample struct { - Tags []testTag + Tags []testTag + TagsToTime map[string]time.Time PreReleases bool Variant string - Regex string + Regex string + CreatedAtSort bool SemverConstraint string @@ -1534,6 +1575,7 @@ func (example SemverOrRegexTagCheckExample) Run() { Variant: example.Variant, SemverConstraint: example.SemverConstraint, Regex: example.Regex, + CreatedAtSort: example.CreatedAtSort, }, } @@ -1612,6 +1654,45 @@ func (example SemverOrRegexTagCheckExample) Run() { ) } + // if SortByCreatedAt is set, we need to return the created date for each tag when the manifest is requested + if example.CreatedAtSort { + manifestRef, err := image.Manifest() + Expect(err).ToNot(HaveOccurred()) + // Mutate ConfigFile such that created at is set to the tag name + expectedTime := example.TagsToTime[tag.ImageName] + config, err := image.ConfigFile() + Expect(err).ToNot(HaveOccurred()) + config.Created = v1.Time{Time: expectedTime} + configBytes, err := json.Marshal(config) + Expect(err).ToNot(HaveOccurred()) + + // Take the SHA256 of config and set to mutatedManifest object + configHash := sha256.Sum256(configBytes) + Expect(err).ToNot(HaveOccurred()) + manifestRef.Config.Digest = v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(configHash[:])} + manifestDigest := manifestRef.Config.Digest + mutatedManifest, err := json.Marshal(manifestRef) + Expect(err).ToNot(HaveOccurred()) + + registryServer.RouteToHandler( + "GET", + "/v2/"+repo.RepositoryStr()+"/manifests/"+tag.Tag, + ghttp.RespondWith(http.StatusOK, mutatedManifest, http.Header{ + "Content-Type": {string(mediaType)}, + "Content-Length": {strconv.Itoa(len(mutatedManifest))}, + "Docker-Content-Digest": {digest.String()}, + }), + ) + + registryServer.RouteToHandler( + "GET", + "/v2/"+repo.RepositoryStr()+"/blobs/"+manifestDigest.String(), + ghttp.RespondWith(http.StatusOK, configBytes, http.Header{ + "Content-Length": {strconv.Itoa(len(configBytes))}, + }), + ) + } + tagVersions[tag.Tag] = resource.Version{ Tag: tag.Tag, Digest: digest.String(), diff --git a/commands/check.go b/commands/check.go index 2ce27c6..67f8ce0 100644 --- a/commands/check.go +++ b/commands/check.go @@ -8,6 +8,7 @@ import ( "regexp" "sort" "strings" + "time" "github.com/Masterminds/semver/v3" resource "github.com/concourse/registry-image-resource" @@ -280,6 +281,7 @@ func checkRepositoryRegex(repo name.Repository, source resource.Source, from *re } tagDigests := map[string]string{} + tagToTimeDigests := map[string]time.Time{} matchedTags := make([]string, 0) for _, identifier := range tags { @@ -300,11 +302,33 @@ func checkRepositoryRegex(repo name.Repository, source resource.Source, from *re continue } + if source.CreatedAtSort { + // Call Get to get the Image and History of the tag + img, err := remote.Image(tagRef, opts...) + if err != nil { + return resource.CheckResponse{}, fmt.Errorf("get remote image: %w", err) + } + + // This calls /blobs/sha256: to get the config file + configFile, err := img.ConfigFile() + if err != nil { + return resource.CheckResponse{}, fmt.Errorf("get remote image config file: %w", err) + } + tagToTimeDigests[identifier] = configFile.Created.Time + } + matchedTags = append(matchedTags, identifier) tagDigests[identifier] = digest.String() } + // If CreatedAtSort is true, sort the matchedTags in descending order by looking up Time in tagToTimeDigests + if source.CreatedAtSort { + sort.Slice(matchedTags, func(i, j int) bool { + return tagToTimeDigests[matchedTags[i]].Before(tagToTimeDigests[matchedTags[j]]) + }) + } + response := resource.CheckResponse{} // Using matchedTags here maintains the order of the response to the list tags call diff --git a/types.go b/types.go index 350fd44..5177130 100644 --- a/types.go +++ b/types.go @@ -95,7 +95,8 @@ type Source struct { Tag Tag `json:"tag,omitempty"` - Regex string `json:"tag_regex,omitempty"` + Regex string `json:"tag_regex,omitempty"` + CreatedAtSort bool `json:"created_at_sort,omitempty"` BasicCredentials AwsCredentials From 6907f074e77b2fb3631704b7809f2622db0609aa Mon Sep 17 00:00:00 2001 From: MD Islam Date: Tue, 19 Mar 2024 16:23:12 -0400 Subject: [PATCH 2/2] add 182 instead of 181 Co-authored-by: Rui Yang Signed-off-by: MD Islam --- check_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/check_test.go b/check_test.go index 6eb598e..b0fd04c 100644 --- a/check_test.go +++ b/check_test.go @@ -830,7 +830,7 @@ var _ = DescribeTable("tracking semver tags", ImageName: "random-3", }, { - Tag: "gem-181-git-6bd8a5e1a2b3", + Tag: "gem-182-git-6bd8a5e1a2b3", ImageName: "random-4", }, { @@ -840,7 +840,7 @@ var _ = DescribeTable("tracking semver tags", }, TagsToTime: map[string]time.Time{ "gem-1338-git-4bd8a5e1a244": time.Date(2024, 1, 4, 5, 0, 0, 0, time.UTC), - "gem-181-git-6bd8a5e1a2b3": time.Date(2024, 1, 4, 0, 0, 0, 0, time.UTC), + "gem-182-git-6bd8a5e1a2b3": time.Date(2024, 1, 4, 0, 0, 0, 0, time.UTC), "gem-1337-git-4bd8a5e1a244": time.Date(2024, 1, 4, 4, 0, 0, 0, time.UTC), }, Regex: "gem-(\\d+)-git-([a-f0-9]{12})", @@ -1659,7 +1659,7 @@ func (example SemverOrRegexTagCheckExample) Run() { manifestRef, err := image.Manifest() Expect(err).ToNot(HaveOccurred()) // Mutate ConfigFile such that created at is set to the tag name - expectedTime := example.TagsToTime[tag.ImageName] + expectedTime := example.TagsToTime[tag.Tag] config, err := image.ConfigFile() Expect(err).ToNot(HaveOccurred()) config.Created = v1.Time{Time: expectedTime}