From 52c71615132e2d749044ffe92bdadc58f21acc0e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 10 Dec 2023 11:55:22 +0000 Subject: [PATCH] fix(deps): update github.com/containers/image/v5 digest to 28a299f Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 12 +- go.sum | 24 +-- .../containers/image/v5/copy/manifest.go | 82 +++++-- .../containers/image/v5/copy/single.go | 10 + .../image/v5/docker/docker_image.go | 3 + .../image/v5/docker/docker_image_dest.go | 10 +- .../image/v5/docker/docker_image_src.go | 15 +- .../image/v5/docker/docker_transport.go | 63 +++++- .../image/v5/internal/manifest/manifest.go | 26 ++- .../image/v5/internal/private/private.go | 1 + .../image/v5/manifest/docker_schema1.go | 15 ++ .../containers/image/v5/manifest/manifest.go | 2 +- .../containers/image/v5/manifest/oci.go | 38 +++- .../image/v5/oci/archive/oci_dest.go | 8 +- .../image/v5/signature/fulcio_cert.go | 3 + .../image/v5/signature/fulcio_cert_stub.go | 28 +++ .../image/v5/signature/internal/rekor_set.go | 3 + .../v5/signature/internal/rekor_set_stub.go | 15 ++ .../image/v5/storage/storage_dest.go | 203 ++++++++++++------ .../image/v5/storage/storage_src.go | 89 ++++++-- .../containers/storage/drivers/driver.go | 11 +- .../storage/drivers/overlay/overlay.go | 6 +- .../github.com/containers/storage/layers.go | 108 +++++++--- .../containers/storage/pkg/chunked/toc/toc.go | 34 +++ vendor/github.com/containers/storage/store.go | 19 +- vendor/github.com/go-openapi/strfmt/time.go | 4 +- vendor/github.com/vbauerster/mpb/v8/README.md | 6 +- .../vbauerster/mpb/v8/decor/decorator.go | 27 ++- vendor/modules.txt | 13 +- 29 files changed, 669 insertions(+), 209 deletions(-) create mode 100644 vendor/github.com/containers/image/v5/signature/fulcio_cert_stub.go create mode 100644 vendor/github.com/containers/image/v5/signature/internal/rekor_set_stub.go create mode 100644 vendor/github.com/containers/storage/pkg/chunked/toc/toc.go diff --git a/go.mod b/go.mod index 6df0532ec34..46cb1a68810 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 github.com/containers/common v0.57.1-0.20231130092720-630c929caef9 - github.com/containers/image/v5 v5.29.1-0.20231120202631-293b00ba7166 + github.com/containers/image/v5 v5.29.1-0.20231206154545-28a299f3d39e github.com/containers/luksy v0.0.0-20231030195837-b5a7f79da98b github.com/containers/ocicrypt v1.1.9 - github.com/containers/storage v1.51.1-0.20231204015418-15c3cb7881e4 + github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5 github.com/cyphar/filepath-securejoin v0.2.4 github.com/docker/distribution v2.8.3+incompatible github.com/docker/docker v24.0.7+incompatible @@ -74,7 +74,7 @@ require ( github.com/go-openapi/loads v0.21.2 // indirect github.com/go-openapi/runtime v0.26.0 // indirect github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/strfmt v0.21.7 // indirect + github.com/go-openapi/strfmt v0.21.8 // indirect github.com/go-openapi/swag v0.22.4 // indirect github.com/go-openapi/validate v0.22.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect @@ -118,14 +118,14 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/sigstore/fulcio v1.4.3 // indirect github.com/sigstore/rekor v1.2.2 // indirect - github.com/sigstore/sigstore v1.7.5 // indirect + github.com/sigstore/sigstore v1.7.6 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect github.com/sylabs/sif/v2 v2.15.0 // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/vbatts/tar-split v0.11.5 // indirect - github.com/vbauerster/mpb/v8 v8.6.2 // indirect + github.com/vbauerster/mpb/v8 v8.7.0 // indirect github.com/vishvananda/netlink v1.2.1-beta.2 // indirect github.com/vishvananda/netns v0.0.4 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect @@ -133,7 +133,7 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.18.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect diff --git a/go.sum b/go.sum index 68e2dfb25fd..1b68724cc8c 100644 --- a/go.sum +++ b/go.sum @@ -56,16 +56,16 @@ github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= github.com/containers/common v0.57.1-0.20231130092720-630c929caef9 h1:56pMgYcYyhTlmPPhRmG34NBmT5S/IwMMmOq0o4LJAMo= github.com/containers/common v0.57.1-0.20231130092720-630c929caef9/go.mod h1:1TyelTjZvU4ZVSq6tGl0ImFlMKIbE8QkzpACQCdcs4U= -github.com/containers/image/v5 v5.29.1-0.20231120202631-293b00ba7166 h1:Dz4ryT8VDKn6U+oWPtsihAV2eG7uFc+LYS7UjHjLcwk= -github.com/containers/image/v5 v5.29.1-0.20231120202631-293b00ba7166/go.mod h1:0uOgAiVgmF8+VCXltRYmncWjkDYc+jFma49NKNz0cS4= +github.com/containers/image/v5 v5.29.1-0.20231206154545-28a299f3d39e h1:YZOMGJyVZkejYSBXh2bsjTTvAownWLE9UWvVQ0KAXRs= +github.com/containers/image/v5 v5.29.1-0.20231206154545-28a299f3d39e/go.mod h1:iVv15HhIYt4p37EemYt9SdFhj+FIZ8AzwKH78qvGMkI= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/luksy v0.0.0-20231030195837-b5a7f79da98b h1:8XvNAm+g7ivwPUkyiHvBs7z356JWpK9a0FDaek86+sY= github.com/containers/luksy v0.0.0-20231030195837-b5a7f79da98b/go.mod h1:menB9p4o5HckgcLW6cO0+dl6+axkVmSqKlrNcratsh4= github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM= github.com/containers/ocicrypt v1.1.9/go.mod h1:dTKx1918d8TDkxXvarscpNVY+lyPakPNFN4jwA9GBys= -github.com/containers/storage v1.51.1-0.20231204015418-15c3cb7881e4 h1:dwGIPF0WNxygqTi1tdGfYSiysu0hSTpy5aaG1zGycVM= -github.com/containers/storage v1.51.1-0.20231204015418-15c3cb7881e4/go.mod h1:pMhG1O3eMGlQKpuEuv7ves+K3BsK8/UJs8ctV5fEaoI= +github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5 h1:eiCkAt+i9BYRjR7KEKPI3iORCSABhY+spM/w8BkI2lo= +github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5/go.mod h1:pMhG1O3eMGlQKpuEuv7ves+K3BsK8/UJs8ctV5fEaoI= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -138,8 +138,8 @@ github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6 github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= +github.com/go-openapi/strfmt v0.21.8 h1:VYBUoKYRLAlgKDrIxR/I0lKrztDQ0tuTDrbhLVP8Erg= +github.com/go-openapi/strfmt v0.21.8/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -372,8 +372,8 @@ github.com/sigstore/fulcio v1.4.3 h1:9JcUCZjjVhRF9fmhVuz6i1RyhCc/EGCD7MOl+iqCJLQ github.com/sigstore/fulcio v1.4.3/go.mod h1:BQPWo7cfxmJwgaHlphUHUpFkp5+YxeJes82oo39m5og= github.com/sigstore/rekor v1.2.2 h1:5JK/zKZvcQpL/jBmHvmFj3YbpDMBQnJQ6ygp8xdF3bY= github.com/sigstore/rekor v1.2.2/go.mod h1:FGnWBGWzeNceJnp0x9eDFd41mI8aQqCjj+Zp0IEs0Qg= -github.com/sigstore/sigstore v1.7.5 h1:ij55dBhLwjICmLTBJZm7SqoQLdsu/oowDanACcJNs48= -github.com/sigstore/sigstore v1.7.5/go.mod h1:9OCmYWhzuq/G4e1cy9m297tuMRJ1LExyrXY3ZC3Zt/s= +github.com/sigstore/sigstore v1.7.6 h1:zB0woXx+3Bp7dk7AjklHF1VhXBdCs84VXkZbp0IHLv8= +github.com/sigstore/sigstore v1.7.6/go.mod h1:FJE+NpEZIs4QKqZl4B2RtaVLVDcDtocAwTiNlexeBkY= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -415,8 +415,8 @@ github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= -github.com/vbauerster/mpb/v8 v8.6.2 h1:9EhnJGQRtvgDVCychJgR96EDCOqgg2NsMuk5JUcX4DA= -github.com/vbauerster/mpb/v8 v8.6.2/go.mod h1:oVJ7T+dib99kZ/VBjoBaC8aPXiSAihnzuKmotuihyFo= +github.com/vbauerster/mpb/v8 v8.7.0 h1:n2LTGyol7qqNBcLQn8FL5Bga2O8CGF75OOYsJVFsfMg= +github.com/vbauerster/mpb/v8 v8.7.0/go.mod h1:0RgdqeTpu6cDbdWeSaDvEvfgm9O598rBnRZ09HKaV0k= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= @@ -480,8 +480,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/github.com/containers/image/v5/copy/manifest.go b/vendor/github.com/containers/image/v5/copy/manifest.go index 6f01cf5cc3b..8844ac8e7eb 100644 --- a/vendor/github.com/containers/image/v5/copy/manifest.go +++ b/vendor/github.com/containers/image/v5/copy/manifest.go @@ -6,8 +6,10 @@ import ( "fmt" "strings" + internalManifest "github.com/containers/image/v5/internal/manifest" "github.com/containers/image/v5/internal/set" "github.com/containers/image/v5/manifest" + compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" @@ -19,8 +21,8 @@ import ( // Include v2s1 signed but not v2s1 unsigned, because docker/distribution requires a signature even if the unsigned MIME type is used. var preferredManifestMIMETypes = []string{manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType} -// ociEncryptionMIMETypes lists manifest MIME types that are known to support OCI encryption. -var ociEncryptionMIMETypes = []string{v1.MediaTypeImageManifest} +// allManifestMIMETypes lists all possible manifest MIME types. +var allManifestMIMETypes = []string{v1.MediaTypeImageManifest, manifest.DockerV2Schema2MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType} // orderedSet is a list of strings (MIME types or platform descriptors in our case), with each string appearing at most once. type orderedSet struct { @@ -51,9 +53,10 @@ type determineManifestConversionInputs struct { destSupportedManifestMIMETypes []string // MIME types supported by the destination, per types.ImageDestination.SupportedManifestMIMETypes() - forceManifestMIMEType string // User’s choice of forced manifest MIME type - requiresOCIEncryption bool // Restrict to manifest formats that can support OCI encryption - cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can + forceManifestMIMEType string // User’s choice of forced manifest MIME type + requestedCompressionFormat *compressiontypes.Algorithm // Compression algorithm to use, if the user _explictily_ requested one. + requiresOCIEncryption bool // Restrict to manifest formats that can support OCI encryption + cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can } // manifestConversionPlan contains the decisions made by determineManifestConversion. @@ -80,41 +83,74 @@ func determineManifestConversion(in determineManifestConversionInputs) (manifest destSupportedManifestMIMETypes = []string{in.forceManifestMIMEType} } + restrictiveCompressionRequired := in.requestedCompressionFormat != nil && !internalManifest.CompressionAlgorithmIsUniversallySupported(*in.requestedCompressionFormat) if len(destSupportedManifestMIMETypes) == 0 { - if !in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(srcType) { + if (!in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(srcType)) && + (!restrictiveCompressionRequired || internalManifest.MIMETypeSupportsCompressionAlgorithm(srcType, *in.requestedCompressionFormat)) { return manifestConversionPlan{ // Anything goes; just use the original as is, do not try any conversions. preferredMIMEType: srcType, otherMIMETypeCandidates: []string{}, }, nil } - destSupportedManifestMIMETypes = ociEncryptionMIMETypes + destSupportedManifestMIMETypes = allManifestMIMETypes } supportedByDest := set.New[string]() for _, t := range destSupportedManifestMIMETypes { - if !in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(t) { - supportedByDest.Add(t) + if in.requiresOCIEncryption && !manifest.MIMETypeSupportsEncryption(t) { + continue } + if restrictiveCompressionRequired && !internalManifest.MIMETypeSupportsCompressionAlgorithm(t, *in.requestedCompressionFormat) { + continue + } + supportedByDest.Add(t) } if supportedByDest.Empty() { - if len(destSupportedManifestMIMETypes) == 0 { // Coverage: This should never happen, empty values were replaced by ociEncryptionMIMETypes + if len(destSupportedManifestMIMETypes) == 0 { // Coverage: This should never happen, empty values were replaced by allManifestMIMETypes return manifestConversionPlan{}, errors.New("internal error: destSupportedManifestMIMETypes is empty") } - // We know, and have verified, that destSupportedManifestMIMETypes is not empty, so encryption must have been involved. - if !in.requiresOCIEncryption { // Coverage: This should never happen, destSupportedManifestMIMETypes was not empty, so we should have filtered for encryption. - return manifestConversionPlan{}, errors.New("internal error: supportedByDest is empty but destSupportedManifestMIMETypes is not, and not encrypting") - } + // We know, and have verified, that destSupportedManifestMIMETypes is not empty, so some filtering of supported MIME types must have been involved. + // destSupportedManifestMIMETypes has three possible origins: if in.forceManifestMIMEType != "" { // 1. forceManifestType specified - return manifestConversionPlan{}, fmt.Errorf("encryption required together with format %s, which does not support encryption", - in.forceManifestMIMEType) + switch { + case in.requiresOCIEncryption && restrictiveCompressionRequired: + return manifestConversionPlan{}, fmt.Errorf("compression using %s, and encryption, required together with format %s, which does not support both", + in.requestedCompressionFormat.Name(), in.forceManifestMIMEType) + case in.requiresOCIEncryption: + return manifestConversionPlan{}, fmt.Errorf("encryption required together with format %s, which does not support encryption", + in.forceManifestMIMEType) + case restrictiveCompressionRequired: + return manifestConversionPlan{}, fmt.Errorf("compression using %s required together with format %s, which does not support it", + in.requestedCompressionFormat.Name(), in.forceManifestMIMEType) + default: + return manifestConversionPlan{}, errors.New("internal error: forceManifestMIMEType was rejected for an unknown reason") + } + } + if len(in.destSupportedManifestMIMETypes) == 0 { // 2. destination accepts anything and we have chosen allManifestTypes + if !restrictiveCompressionRequired { + // Coverage: This should never happen. + // If we have not rejected for encryption reasons, we must have rejected due to encryption, but + // allManifestTypes includes OCI, which supports encryption. + return manifestConversionPlan{}, errors.New("internal error: in.destSupportedManifestMIMETypes is empty but supportedByDest is empty as well") + } + // This can legitimately happen when the user asks for completely unsupported formats like Bzip2 or Xz. + return manifestConversionPlan{}, fmt.Errorf("compression using %s required, but none of the known manifest formats support it", in.requestedCompressionFormat.Name()) } - if len(in.destSupportedManifestMIMETypes) == 0 { // 2. destination accepts anything and we have chosen ociEncryptionMIMETypes - // Coverage: This should never happen, ociEncryptionMIMETypes all support encryption - return manifestConversionPlan{}, errors.New("internal error: in.destSupportedManifestMIMETypes is empty but supportedByDest is empty as well") + // 3. destination accepts a restricted list of mime types + destMIMEList := strings.Join(destSupportedManifestMIMETypes, ", ") + switch { + case in.requiresOCIEncryption && restrictiveCompressionRequired: + return manifestConversionPlan{}, fmt.Errorf("compression using %s, and encryption, required but the destination only supports MIME types [%s], none of which support both", + in.requestedCompressionFormat.Name(), destMIMEList) + case in.requiresOCIEncryption: + return manifestConversionPlan{}, fmt.Errorf("encryption required but the destination only supports MIME types [%s], none of which support encryption", + destMIMEList) + case restrictiveCompressionRequired: + return manifestConversionPlan{}, fmt.Errorf("compression using %s required but the destination only supports MIME types [%s], none of which support it", + in.requestedCompressionFormat.Name(), destMIMEList) + default: // Coverage: This should never happen, we only filter for in.requiresOCIEncryption || restrictiveCompressionRequired + return manifestConversionPlan{}, errors.New("internal error: supportedByDest is empty but destSupportedManifestMIMETypes is not, and we are neither encrypting nor requiring a restrictive compression algorithm") } - // 3. destination does not support encryption. - return manifestConversionPlan{}, fmt.Errorf("encryption required but the destination only supports MIME types [%s], none of which support encryption", - strings.Join(destSupportedManifestMIMETypes, ", ")) } // destSupportedManifestMIMETypes is a static guess; a particular registry may still only support a subset of the types. @@ -156,7 +192,7 @@ func determineManifestConversion(in determineManifestConversionInputs) (manifest } logrus.Debugf("Manifest has MIME type %s, ordered candidate list [%s]", srcType, strings.Join(prioritizedTypes.list, ", ")) - if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes and supportedByDest, which is a subset, is not empty (or we would have exited above), so this should never happen. + if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes and supportedByDest, which is a subset, is not empty (or we would have exited above), so this should never happen. return manifestConversionPlan{}, errors.New("Internal error: no candidate MIME types") } res := manifestConversionPlan{ diff --git a/vendor/github.com/containers/image/v5/copy/single.go b/vendor/github.com/containers/image/v5/copy/single.go index 67ca43f7bcf..9003965c956 100644 --- a/vendor/github.com/containers/image/v5/copy/single.go +++ b/vendor/github.com/containers/image/v5/copy/single.go @@ -20,6 +20,7 @@ import ( compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" + chunkedToc "github.com/containers/storage/pkg/chunked/toc" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" @@ -167,6 +168,7 @@ func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.Unpar srcMIMEType: ic.src.ManifestMIMEType, destSupportedManifestMIMETypes: ic.c.dest.SupportedManifestMIMETypes(), forceManifestMIMEType: c.options.ForceManifestMIMEType, + requestedCompressionFormat: ic.compressionFormat, requiresOCIEncryption: destRequiresOciEncryption, cannotModifyManifestReason: ic.cannotModifyManifestReason, }) @@ -693,6 +695,13 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to requiredCompression = ic.compressionFormat originalCompression = srcInfo.CompressionAlgorithm } + + // Check if we have a chunked layer in storage that's based on that blob. These layers are stored by their TOC digest. + tocDigest, err := chunkedToc.GetTOCDigest(srcInfo.Annotations) + if err != nil { + return types.BlobInfo{}, "", err + } + reused, reusedBlob, err := ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{ Cache: ic.c.blobInfoCache, CanSubstitute: canSubstitute, @@ -701,6 +710,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to SrcRef: srcRef, RequiredCompression: requiredCompression, OriginalCompression: originalCompression, + TOCDigest: tocDigest, }) if err != nil { return types.BlobInfo{}, "", fmt.Errorf("trying to reuse blob %s at destination: %w", srcInfo.Digest, err) diff --git a/vendor/github.com/containers/image/v5/docker/docker_image.go b/vendor/github.com/containers/image/v5/docker/docker_image.go index 42bbfd95ee2..93160480ea2 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image.go @@ -123,6 +123,9 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef if !ok { return "", errors.New("ref must be a dockerReference") } + if dr.isUnknownDigest { + return "", fmt.Errorf("docker: reference %q is for unknown digest case; cannot get digest", dr.StringWithinTransport()) + } tagOrDigest, err := dr.tagOrDigest() if err != nil { diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go index 774068c2769..a9a36f0a34a 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go @@ -452,7 +452,15 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, // but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { var refTail string - if instanceDigest != nil { + // If d.ref.isUnknownDigest=true, then we push without a tag, so get the + // digest that will be used + if d.ref.isUnknownDigest { + digest, err := manifest.Digest(m) + if err != nil { + return err + } + refTail = digest.String() + } else if instanceDigest != nil { // If the instanceDigest is provided, then use it as the refTail, because the reference, // whether it includes a tag or a digest, refers to the list as a whole, and not this // particular instance. diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go index 231d5d21240..f9d4d6030f0 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go @@ -38,8 +38,8 @@ type dockerImageSource struct { impl.DoesNotAffectLayerInfosForCopy stubs.ImplementsGetBlobAt - logicalRef dockerReference // The reference the user requested. - physicalRef dockerReference // The actual reference we are accessing (possibly a mirror) + logicalRef dockerReference // The reference the user requested. This must satisfy !isUnknownDigest + physicalRef dockerReference // The actual reference we are accessing (possibly a mirror). This must satisfy !isUnknownDigest c *dockerClient // State cachedManifest []byte // nil if not loaded yet @@ -48,7 +48,12 @@ type dockerImageSource struct { // newImageSource creates a new ImageSource for the specified image reference. // The caller must call .Close() on the returned ImageSource. +// The caller must ensure !ref.isUnknownDigest. func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) { + if ref.isUnknownDigest { + return nil, fmt.Errorf("reading images from docker: reference %q without a tag or digest is not supported", ref.StringWithinTransport()) + } + registryConfig, err := loadRegistryConfiguration(sys) if err != nil { return nil, err @@ -121,7 +126,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerRef // The caller must call .Close() on the returned ImageSource. func newImageSourceAttempt(ctx context.Context, sys *types.SystemContext, logicalRef dockerReference, pullSource sysregistriesv2.PullSource, registryConfig *registryConfiguration) (*dockerImageSource, error) { - physicalRef, err := newReference(pullSource.Reference) + physicalRef, err := newReference(pullSource.Reference, false) if err != nil { return nil, err } @@ -591,6 +596,10 @@ func (s *dockerImageSource) getSignaturesFromSigstoreAttachments(ctx context.Con // deleteImage deletes the named image from the registry, if supported. func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerReference) error { + if ref.isUnknownDigest { + return fmt.Errorf("Docker reference without a tag or digest cannot be deleted") + } + registryConfig, err := loadRegistryConfiguration(sys) if err != nil { return err diff --git a/vendor/github.com/containers/image/v5/docker/docker_transport.go b/vendor/github.com/containers/image/v5/docker/docker_transport.go index 6ae8491594a..1c89302f46b 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_transport.go +++ b/vendor/github.com/containers/image/v5/docker/docker_transport.go @@ -12,6 +12,11 @@ import ( "github.com/containers/image/v5/types" ) +// UnknownDigestSuffix can be appended to a reference when the caller +// wants to push an image without a tag or digest. +// NewReferenceUnknownDigest() is called when this const is detected. +const UnknownDigestSuffix = "@@unknown-digest@@" + func init() { transports.Register(Transport) } @@ -43,7 +48,8 @@ func (t dockerTransport) ValidatePolicyConfigurationScope(scope string) error { // dockerReference is an ImageReference for Docker images. type dockerReference struct { - ref reference.Named // By construction we know that !reference.IsNameOnly(ref) + ref reference.Named // By construction we know that !reference.IsNameOnly(ref) unless isUnknownDigest=true + isUnknownDigest bool } // ParseReference converts a string, which should not start with the ImageTransport.Name prefix, into an Docker ImageReference. @@ -51,23 +57,46 @@ func ParseReference(refString string) (types.ImageReference, error) { if !strings.HasPrefix(refString, "//") { return nil, fmt.Errorf("docker: image reference %s does not start with //", refString) } + // Check if ref has UnknownDigestSuffix suffixed to it + unknownDigest := false + if strings.HasSuffix(refString, UnknownDigestSuffix) { + unknownDigest = true + refString = strings.TrimSuffix(refString, UnknownDigestSuffix) + } ref, err := reference.ParseNormalizedNamed(strings.TrimPrefix(refString, "//")) if err != nil { return nil, err } + + if unknownDigest { + if !reference.IsNameOnly(ref) { + return nil, fmt.Errorf("docker: image reference %q has unknown digest set but it contains either a tag or digest", ref.String()+UnknownDigestSuffix) + } + return NewReferenceUnknownDigest(ref) + } + ref = reference.TagNameOnly(ref) return NewReference(ref) } // NewReference returns a Docker reference for a named reference. The reference must satisfy !reference.IsNameOnly(). func NewReference(ref reference.Named) (types.ImageReference, error) { - return newReference(ref) + return newReference(ref, false) +} + +// NewReferenceUnknownDigest returns a Docker reference for a named reference, which can be used to write images without setting +// a tag on the registry. The reference must satisfy reference.IsNameOnly() +func NewReferenceUnknownDigest(ref reference.Named) (types.ImageReference, error) { + return newReference(ref, true) } // newReference returns a dockerReference for a named reference. -func newReference(ref reference.Named) (dockerReference, error) { - if reference.IsNameOnly(ref) { - return dockerReference{}, fmt.Errorf("Docker reference %s has neither a tag nor a digest", reference.FamiliarString(ref)) +func newReference(ref reference.Named, unknownDigest bool) (dockerReference, error) { + if reference.IsNameOnly(ref) && !unknownDigest { + return dockerReference{}, fmt.Errorf("Docker reference %s is not for an unknown digest case; tag or digest is needed", reference.FamiliarString(ref)) + } + if !reference.IsNameOnly(ref) && unknownDigest { + return dockerReference{}, fmt.Errorf("Docker reference %s is for an unknown digest case but reference has a tag or digest", reference.FamiliarString(ref)) } // A github.com/distribution/reference value can have a tag and a digest at the same time! // The docker/distribution API does not really support that (we can’t ask for an image with a specific @@ -81,7 +110,8 @@ func newReference(ref reference.Named) (dockerReference, error) { } return dockerReference{ - ref: ref, + ref: ref, + isUnknownDigest: unknownDigest, }, nil } @@ -95,7 +125,11 @@ func (ref dockerReference) Transport() types.ImageTransport { // e.g. default attribute values omitted by the user may be filled in the return value, or vice versa. // WARNING: Do not use the return value in the UI to describe an image, it does not contain the Transport().Name() prefix. func (ref dockerReference) StringWithinTransport() string { - return "//" + reference.FamiliarString(ref.ref) + famString := "//" + reference.FamiliarString(ref.ref) + if ref.isUnknownDigest { + return famString + UnknownDigestSuffix + } + return famString } // DockerReference returns a Docker reference associated with this reference @@ -113,6 +147,9 @@ func (ref dockerReference) DockerReference() reference.Named { // not required/guaranteed that it will be a valid input to Transport().ParseReference(). // Returns "" if configuration identities for these references are not supported. func (ref dockerReference) PolicyConfigurationIdentity() string { + if ref.isUnknownDigest { + return ref.ref.Name() + } res, err := policyconfiguration.DockerReferenceIdentity(ref.ref) if res == "" || err != nil { // Coverage: Should never happen, NewReference above should refuse values which could cause a failure. panic(fmt.Sprintf("Internal inconsistency: policyconfiguration.DockerReferenceIdentity returned %#v, %v", res, err)) @@ -126,7 +163,13 @@ func (ref dockerReference) PolicyConfigurationIdentity() string { // It is STRONGLY recommended for the first element, if any, to be a prefix of PolicyConfigurationIdentity(), // and each following element to be a prefix of the element preceding it. func (ref dockerReference) PolicyConfigurationNamespaces() []string { - return policyconfiguration.DockerReferenceNamespaces(ref.ref) + namespaces := policyconfiguration.DockerReferenceNamespaces(ref.ref) + if ref.isUnknownDigest { + if len(namespaces) != 0 && namespaces[0] == ref.ref.Name() { + namespaces = namespaces[1:] + } + } + return namespaces } // NewImage returns a types.ImageCloser for this reference, possibly specialized for this ImageTransport. @@ -163,6 +206,10 @@ func (ref dockerReference) tagOrDigest() (string, error) { if ref, ok := ref.ref.(reference.NamedTagged); ok { return ref.Tag(), nil } + + if ref.isUnknownDigest { + return "", fmt.Errorf("Docker reference %q is for an unknown digest case, has neither a digest nor a tag", reference.FamiliarString(ref.ref)) + } // This should not happen, NewReference above refuses reference.IsNameOnly values. return "", fmt.Errorf("Internal inconsistency: Reference %s unexpectedly has neither a digest nor a tag", reference.FamiliarString(ref.ref)) } diff --git a/vendor/github.com/containers/image/v5/internal/manifest/manifest.go b/vendor/github.com/containers/image/v5/internal/manifest/manifest.go index 1dbcc14182a..6f7bc8bbe6d 100644 --- a/vendor/github.com/containers/image/v5/internal/manifest/manifest.go +++ b/vendor/github.com/containers/image/v5/internal/manifest/manifest.go @@ -3,6 +3,7 @@ package manifest import ( "encoding/json" + compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/libtrust" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -14,7 +15,7 @@ import ( const ( // DockerV2Schema1MediaType MIME type represents Docker manifest schema 1 DockerV2Schema1MediaType = "application/vnd.docker.distribution.manifest.v1+json" - // DockerV2Schema1MediaType MIME type represents Docker manifest schema 1 with a JWS signature + // DockerV2Schema1SignedMediaType MIME type represents Docker manifest schema 1 with a JWS signature DockerV2Schema1SignedMediaType = "application/vnd.docker.distribution.manifest.v1+prettyjws" // DockerV2Schema2MediaType MIME type represents Docker manifest schema 2 DockerV2Schema2MediaType = "application/vnd.docker.distribution.manifest.v2+json" @@ -165,3 +166,26 @@ func NormalizedMIMEType(input string) string { return DockerV2Schema1SignedMediaType } } + +// CompressionAlgorithmIsUniversallySupported returns true if MIMETypeSupportsCompressionAlgorithm(mimeType, algo) returns true for all mimeType values. +func CompressionAlgorithmIsUniversallySupported(algo compressiontypes.Algorithm) bool { + switch algo.Name() { // Should this use InternalUnstableUndocumentedMIMEQuestionMark() ? + case compressiontypes.GzipAlgorithmName: + return true + default: + return false + } +} + +// MIMETypeSupportsCompressionAlgorithm returns true if mimeType can represent algo. +func MIMETypeSupportsCompressionAlgorithm(mimeType string, algo compressiontypes.Algorithm) bool { + if CompressionAlgorithmIsUniversallySupported(algo) { + return true + } + switch algo.Name() { // Should this use InternalUnstableUndocumentedMIMEQuestionMark() ? + case compressiontypes.ZstdAlgorithmName, compressiontypes.ZstdChunkedAlgorithmName: + return mimeType == imgspecv1.MediaTypeImageManifest + default: // Includes Bzip2AlgorithmName and XzAlgorithmName, which are defined names but are not supported anywhere + return false + } +} diff --git a/vendor/github.com/containers/image/v5/internal/private/private.go b/vendor/github.com/containers/image/v5/internal/private/private.go index 95d561fcdd8..72b574a5bdf 100644 --- a/vendor/github.com/containers/image/v5/internal/private/private.go +++ b/vendor/github.com/containers/image/v5/internal/private/private.go @@ -117,6 +117,7 @@ type TryReusingBlobOptions struct { EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented. LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise. SrcRef reference.Named // A reference to the source image that contains the input blob. + TOCDigest *digest.Digest // If specified, the blob can be looked up in the destination also by its TOC digest. } // ReusedBlob is information about a blob reused in a destination. diff --git a/vendor/github.com/containers/image/v5/manifest/docker_schema1.go b/vendor/github.com/containers/image/v5/manifest/docker_schema1.go index a80af701afd..762815570cd 100644 --- a/vendor/github.com/containers/image/v5/manifest/docker_schema1.go +++ b/vendor/github.com/containers/image/v5/manifest/docker_schema1.go @@ -10,6 +10,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/manifest" "github.com/containers/image/v5/internal/set" + compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/regexp" "github.com/docker/docker/api/types/versions" @@ -142,6 +143,15 @@ func (m *Schema1) LayerInfos() []LayerInfo { return layers } +const fakeSchema1MIMEType = DockerV2Schema2LayerMediaType // Used only in schema1CompressionMIMETypeSets +var schema1CompressionMIMETypeSets = []compressionMIMETypeSet{ + { + mtsUncompressed: fakeSchema1MIMEType, + compressiontypes.GzipAlgorithmName: fakeSchema1MIMEType, + compressiontypes.ZstdAlgorithmName: mtsUnsupportedMIMEType, + }, +} + // UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls), in order (the root layer first, and then successive layered layers) func (m *Schema1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { // Our LayerInfos includes empty layers (where m.ExtractedV1Compatibility[].ThrowAway), so expect them to be included here as well. @@ -150,6 +160,11 @@ func (m *Schema1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { } m.FSLayers = make([]Schema1FSLayers, len(layerInfos)) for i, info := range layerInfos { + // There are no MIME types in schema1, but we do a “conversion” here to reject unsupported compression algorithms, + // in a way that is consistent with the other schema implementations. + if _, err := updatedMIMEType(schema1CompressionMIMETypeSets, fakeSchema1MIMEType, info); err != nil { + return fmt.Errorf("preparing updated manifest, layer %q: %w", info.Digest, err) + } // (docker push) sets up m.ExtractedV1Compatibility[].{Id,Parent} based on values of info.Digest, // but (docker pull) ignores them in favor of computing DiffIDs from uncompressed data, except verifying the child->parent links and uniqueness. // So, we don't bother recomputing the IDs in m.History.V1Compatibility. diff --git a/vendor/github.com/containers/image/v5/manifest/manifest.go b/vendor/github.com/containers/image/v5/manifest/manifest.go index 959aac935eb..828b8da0b7f 100644 --- a/vendor/github.com/containers/image/v5/manifest/manifest.go +++ b/vendor/github.com/containers/image/v5/manifest/manifest.go @@ -16,7 +16,7 @@ import ( const ( // DockerV2Schema1MediaType MIME type represents Docker manifest schema 1 DockerV2Schema1MediaType = manifest.DockerV2Schema1MediaType - // DockerV2Schema1MediaType MIME type represents Docker manifest schema 1 with a JWS signature + // DockerV2Schema1SignedMediaType MIME type represents Docker manifest schema 1 with a JWS signature DockerV2Schema1SignedMediaType = manifest.DockerV2Schema1SignedMediaType // DockerV2Schema2MediaType MIME type represents Docker manifest schema 2 DockerV2Schema2MediaType = manifest.DockerV2Schema2MediaType diff --git a/vendor/github.com/containers/image/v5/manifest/oci.go b/vendor/github.com/containers/image/v5/manifest/oci.go index a85641c36ad..6d5acb45d85 100644 --- a/vendor/github.com/containers/image/v5/manifest/oci.go +++ b/vendor/github.com/containers/image/v5/manifest/oci.go @@ -9,6 +9,7 @@ import ( compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" ociencspec "github.com/containers/ocicrypt/spec" + chunkedToc "github.com/containers/storage/pkg/chunked/toc" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -235,7 +236,7 @@ func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*type } // ImageID computes an ID which can uniquely identify this image by its contents. -func (m *OCI1) ImageID([]digest.Digest) (string, error) { +func (m *OCI1) ImageID(diffIDs []digest.Digest) (string, error) { // The way m.Config.Digest “uniquely identifies” an image is // by containing RootFS.DiffIDs, which identify the layers of the image. // For non-image artifacts, the we can’t expect the config to change @@ -259,9 +260,44 @@ func (m *OCI1) ImageID([]digest.Digest) (string, error) { if err := m.Config.Digest.Validate(); err != nil { return "", err } + + // If there is any layer that is using partial content, we calculate the image ID + // in a different way since the diffID cannot be validated as for regular pulled images. + for _, layer := range m.Layers { + toc, err := chunkedToc.GetTOCDigest(layer.Annotations) + if err != nil { + return "", fmt.Errorf("error looking up annotation for layer %q: %w", layer.Digest, err) + } + if toc != nil { + return m.calculateImageIDForPartialImage(diffIDs) + } + } + return m.Config.Digest.Hex(), nil } +func (m *OCI1) calculateImageIDForPartialImage(diffIDs []digest.Digest) (string, error) { + newID := digest.Canonical.Digester() + for i, layer := range m.Layers { + diffID := diffIDs[i] + _, err := newID.Hash().Write([]byte(diffID.Hex())) + if err != nil { + return "", fmt.Errorf("error writing diffID %q: %w", diffID, err) + } + toc, err := chunkedToc.GetTOCDigest(layer.Annotations) + if err != nil { + return "", fmt.Errorf("error looking up annotation for layer %q: %w", layer.Digest, err) + } + if toc != nil { + _, err = newID.Hash().Write([]byte(toc.Hex())) + if err != nil { + return "", fmt.Errorf("error writing TOC %q: %w", toc, err) + } + } + } + return newID.Digest().Hex(), nil +} + // CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image // (and the code can handle that). // NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted diff --git a/vendor/github.com/containers/image/v5/oci/archive/oci_dest.go b/vendor/github.com/containers/image/v5/oci/archive/oci_dest.go index 8386c47a3fc..6ca618e3519 100644 --- a/vendor/github.com/containers/image/v5/oci/archive/oci_dest.go +++ b/vendor/github.com/containers/image/v5/oci/archive/oci_dest.go @@ -13,6 +13,7 @@ import ( "github.com/containers/image/v5/internal/signature" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/idtools" digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" ) @@ -169,10 +170,15 @@ func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedTopleve // tar converts the directory at src and saves it to dst func tarDirectory(src, dst string) error { // input is a stream of bytes from the archive of the directory at path - input, err := archive.Tar(src, archive.Uncompressed) + input, err := archive.TarWithOptions(src, &archive.TarOptions{ + Compression: archive.Uncompressed, + // Don’t include the data about the user account this code is running under. + ChownOpts: &idtools.IDPair{UID: 0, GID: 0}, + }) if err != nil { return fmt.Errorf("retrieving stream of bytes from %q: %w", src, err) } + defer input.Close() // creates the tar file outFile, err := os.Create(dst) diff --git a/vendor/github.com/containers/image/v5/signature/fulcio_cert.go b/vendor/github.com/containers/image/v5/signature/fulcio_cert.go index ef5d3df6f04..c11fa46a9d9 100644 --- a/vendor/github.com/containers/image/v5/signature/fulcio_cert.go +++ b/vendor/github.com/containers/image/v5/signature/fulcio_cert.go @@ -1,3 +1,6 @@ +//go:build !containers_image_fulcio_stub +// +build !containers_image_fulcio_stub + package signature import ( diff --git a/vendor/github.com/containers/image/v5/signature/fulcio_cert_stub.go b/vendor/github.com/containers/image/v5/signature/fulcio_cert_stub.go new file mode 100644 index 00000000000..ee79b031ddf --- /dev/null +++ b/vendor/github.com/containers/image/v5/signature/fulcio_cert_stub.go @@ -0,0 +1,28 @@ +//go:build containers_image_fulcio_stub +// +build containers_image_fulcio_stub + +package signature + +import ( + "crypto" + "crypto/ecdsa" + "crypto/x509" + "errors" +) + +type fulcioTrustRoot struct { + caCertificates *x509.CertPool + oidcIssuer string + subjectEmail string +} + +func (f *fulcioTrustRoot) validate() error { + return errors.New("fulcio disabled at compile-time") +} + +func verifyRekorFulcio(rekorPublicKey *ecdsa.PublicKey, fulcioTrustRoot *fulcioTrustRoot, untrustedRekorSET []byte, + untrustedCertificateBytes []byte, untrustedIntermediateChainBytes []byte, untrustedBase64Signature string, + untrustedPayloadBytes []byte) (crypto.PublicKey, error) { + return nil, errors.New("fulcio diabled at compile-time") + +} diff --git a/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go b/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go index d439b5f7a7d..d86e98a45b8 100644 --- a/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go +++ b/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go @@ -1,3 +1,6 @@ +//go:build !containers_image_rekor_stub +// +build !containers_image_rekor_stub + package internal import ( diff --git a/vendor/github.com/containers/image/v5/signature/internal/rekor_set_stub.go b/vendor/github.com/containers/image/v5/signature/internal/rekor_set_stub.go new file mode 100644 index 00000000000..7c121cc2eec --- /dev/null +++ b/vendor/github.com/containers/image/v5/signature/internal/rekor_set_stub.go @@ -0,0 +1,15 @@ +//go:build containers_image_rekor_stub +// +build containers_image_rekor_stub + +package internal + +import ( + "crypto/ecdsa" + "time" +) + +// VerifyRekorSET verifies that unverifiedRekorSET is correctly signed by publicKey and matches the rest of the data. +// Returns bundle upload time on success. +func VerifyRekorSET(publicKey *ecdsa.PublicKey, unverifiedRekorSET []byte, unverifiedKeyOrCertBytes []byte, unverifiedBase64Signature string, unverifiedPayloadBytes []byte) (time.Time, error) { + return time.Time{}, NewInvalidSignatureError("rekor disabled at compile-time") +} diff --git a/vendor/github.com/containers/image/v5/storage/storage_dest.go b/vendor/github.com/containers/image/v5/storage/storage_dest.go index 07e1d5e1f9e..bbbff6cf98a 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_dest.go +++ b/vendor/github.com/containers/image/v5/storage/storage_dest.go @@ -77,13 +77,13 @@ type storageImageDestination struct { indexToStorageID map[int]*string // All accesses to below data are protected by `lock` which is made // *explicit* in the code. - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) - indexToAddedLayerInfo map[int]addedLayerInfo // Mapping from layer (by index) to blob to add to the image - blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer - diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output + uncompressedOrTocDigest map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs or TOC IDs. + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) + indexToAddedLayerInfo map[int]addedLayerInfo // Mapping from layer (by index) to blob to add to the image + blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer + diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output } // addedLayerInfo records data about a layer to use in this image. @@ -117,18 +117,18 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* HasThreadSafePutBlob: true, }), - imageRef: imageRef, - directory: directory, - signatureses: make(map[digest.Digest][]byte), - blobDiffIDs: make(map[digest.Digest]digest.Digest), - blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, - SignaturesSizes: make(map[digest.Digest][]int), - indexToStorageID: make(map[int]*string), - indexToAddedLayerInfo: make(map[int]addedLayerInfo), - diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + uncompressedOrTocDigest: make(map[digest.Digest]digest.Digest), + blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), + indexToStorageID: make(map[int]*string), + indexToAddedLayerInfo: make(map[int]addedLayerInfo), + diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), } dest.Compat = impl.AddCompat(dest) return dest, nil @@ -227,7 +227,7 @@ func (s *storageImageDestination) putBlobToPendingFile(stream io.Reader, blobinf // Record information about the blob. s.lock.Lock() - s.blobDiffIDs[blobDigest] = diffID.Digest() + s.uncompressedOrTocDigest[blobDigest] = diffID.Digest() s.fileSizes[blobDigest] = counter.Count s.filenames[blobDigest] = filename s.lock.Unlock() @@ -289,7 +289,7 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces blobDigest := srcInfo.Digest s.lock.Lock() - s.blobDiffIDs[blobDigest] = blobDigest + s.uncompressedOrTocDigest[blobDigest] = blobDigest s.fileSizes[blobDigest] = 0 s.filenames[blobDigest] = "" s.diffOutputs[blobDigest] = out @@ -321,7 +321,7 @@ func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, }) } -// tryReusingBlobAsPending implements TryReusingBlobWithOptions for (digest, size or -1), filling s.blobDiffIDs and other metadata. +// tryReusingBlobAsPending implements TryReusingBlobWithOptions for (digest, size or -1), filling s.uncompressedOrTocDigest and other metadata. // The caller must arrange the blob to be eventually committed using s.commitLayer(). func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, size int64, options *private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { // lock the entire method as it executes fairly quickly @@ -335,7 +335,7 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, return false, private.ReusedBlob{}, fmt.Errorf(`looking for compressed layers with digest %q and labels: %w`, digest, err) } else if err == nil { // Record the uncompressed value so that we can use it to calculate layer IDs. - s.blobDiffIDs[digest] = aLayer.UncompressedDigest() + s.uncompressedOrTocDigest[digest] = aLayer.UncompressedDigest() s.blobAdditionalLayer[digest] = aLayer return true, private.ReusedBlob{ Digest: digest, @@ -366,7 +366,7 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, } if len(layers) > 0 { // Save this for completeness. - s.blobDiffIDs[digest] = layers[0].UncompressedDigest + s.uncompressedOrTocDigest[digest] = layers[0].UncompressedDigest return true, private.ReusedBlob{ Digest: digest, Size: layers[0].UncompressedSize, @@ -380,7 +380,7 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, } if len(layers) > 0 { // Record the uncompressed value so that we can use it to calculate layer IDs. - s.blobDiffIDs[digest] = layers[0].UncompressedDigest + s.uncompressedOrTocDigest[digest] = layers[0].UncompressedDigest return true, private.ReusedBlob{ Digest: digest, Size: layers[0].CompressedSize, @@ -398,7 +398,7 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, } if len(layers) > 0 { if size != -1 { - s.blobDiffIDs[digest] = layers[0].UncompressedDigest + s.uncompressedOrTocDigest[digest] = layers[0].UncompressedDigest return true, private.ReusedBlob{ Digest: digest, Size: size, @@ -407,7 +407,7 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, if !options.CanSubstitute { return false, private.ReusedBlob{}, fmt.Errorf("Internal error: options.CanSubstitute was expected to be true for blob with digest %s", digest) } - s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest + s.uncompressedOrTocDigest[uncompressedDigest] = layers[0].UncompressedDigest return true, private.ReusedBlob{ Digest: uncompressedDigest, Size: layers[0].UncompressedSize, @@ -416,6 +416,25 @@ func (s *storageImageDestination) tryReusingBlobAsPending(digest digest.Digest, } } + tocDigest := digest + if options.TOCDigest != nil { + tocDigest = *options.TOCDigest + } + + // Check if we have a chunked layer in storage with the same TOC digest. + layers, err = s.imageRef.transport.store.LayersByTOCDigest(tocDigest) + if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { + return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with TOC digest %q: %w`, tocDigest, err) + } + if len(layers) > 0 { + // Save this for completeness. + s.uncompressedOrTocDigest[digest] = layers[0].TOCDigest + return true, private.ReusedBlob{ + Digest: layers[0].TOCDigest, + Size: layers[0].UncompressedSize, + }, nil + } + // Nope, we don't have it. return false, private.ReusedBlob{}, nil } @@ -438,16 +457,20 @@ func (s *storageImageDestination) computeID(m manifest.Manifest) string { continue } blobSum := m.FSLayers[i].BlobSum - diffID, ok := s.blobDiffIDs[blobSum] + diffID, ok := s.uncompressedOrTocDigest[blobSum] if !ok { logrus.Infof("error looking up diffID for layer %q", blobSum.String()) return "" } diffIDs = append([]digest.Digest{diffID}, diffIDs...) } - case *manifest.Schema2, *manifest.OCI1: - // We know the ID calculation for these formats doesn't actually use the diffIDs, - // so we don't need to populate the diffID list. + case *manifest.Schema2: + // We know the ID calculation doesn't actually use the diffIDs, so we don't need to populate + // the diffID list. + case *manifest.OCI1: + for _, l := range m.Layers { + diffIDs = append(diffIDs, l.Digest) + } default: return "" } @@ -518,7 +541,7 @@ func (s *storageImageDestination) queueOrCommit(index int, info addedLayerInfo) } s.lock.Unlock() // Note: commitLayer locks on-demand. - if err := s.commitLayer(index, info, -1); err != nil { + if stopQueue, err := s.commitLayer(index, info, -1); stopQueue || err != nil { return err } s.lock.Lock() @@ -532,18 +555,32 @@ func (s *storageImageDestination) queueOrCommit(index int, info addedLayerInfo) return nil } +// getDiffIDOrTOCDigest returns the diffID for the specified digest or the digest for the TOC, if known. +func (s *storageImageDestination) getDiffIDOrTOCDigest(uncompressedDigest digest.Digest) (digest.Digest, bool) { + s.lock.Lock() + defer s.lock.Unlock() + + if d, found := s.diffOutputs[uncompressedDigest]; found { + return d.TOCDigest, found + } + d, found := s.uncompressedOrTocDigest[uncompressedDigest] + return d, found +} + // commitLayer commits the specified layer with the given index to the storage. -// size can usually be -1; it can be provided if the layer is not known to be already present in blobDiffIDs. +// size can usually be -1; it can be provided if the layer is not known to be already present in uncompressedOrTocDigest. +// +// If the layer cannot be committed yet, the function returns (true, nil). // // Note that the previous layer is expected to already be committed. // // Caution: this function must be called without holding `s.lock`. Callers // must guarantee that, at any given time, at most one goroutine may execute // `commitLayer()`. -func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, size int64) error { +func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, size int64) (bool, error) { // Already committed? Return early. if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted { - return nil + return false, nil } // Start with an empty string or the previous layer ID. Note that @@ -557,68 +594,96 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si // Carry over the previous ID for empty non-base layers. if info.emptyLayer { s.indexToStorageID[index] = &lastLayer - return nil + return false, nil } // Check if there's already a layer with the ID that we'd give to the result of applying // this layer blob to its parent, if it has one, or the blob's hex value otherwise. - s.lock.Lock() - diffID, haveDiffID := s.blobDiffIDs[info.digest] - s.lock.Unlock() - if !haveDiffID { + // The diffIDOrTOCDigest refers either to the DiffID or the digest of the TOC. + diffIDOrTOCDigest, haveDiffIDOrTOCDigest := s.getDiffIDOrTOCDigest(info.digest) + if !haveDiffIDOrTOCDigest { // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), // or to even check if we had it. // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller // that relies on using a blob digest that has never been seen by the store had better call // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only // so far we are going to accommodate that (if we should be doing that at all). - logrus.Debugf("looking for diffID for blob %+v", info.digest) + logrus.Debugf("looking for diffID or TOC digest for blob %+v", info.digest) // Use tryReusingBlobAsPending, not the top-level TryReusingBlobWithOptions, to prevent recursion via queueOrCommit. has, _, err := s.tryReusingBlobAsPending(info.digest, size, &private.TryReusingBlobOptions{ Cache: none.NoCache, CanSubstitute: false, }) if err != nil { - return fmt.Errorf("checking for a layer based on blob %q: %w", info.digest.String(), err) + return false, fmt.Errorf("checking for a layer based on blob %q: %w", info.digest.String(), err) } if !has { - return fmt.Errorf("error determining uncompressed digest for blob %q", info.digest.String()) + return false, fmt.Errorf("error determining uncompressed digest or TOC digest for blob %q", info.digest.String()) } - diffID, haveDiffID = s.blobDiffIDs[info.digest] - if !haveDiffID { - return fmt.Errorf("we have blob %q, but don't know its uncompressed digest", info.digest.String()) + diffIDOrTOCDigest, haveDiffIDOrTOCDigest = s.getDiffIDOrTOCDigest(info.digest) + if !haveDiffIDOrTOCDigest { + return false, fmt.Errorf("we have blob %q, but don't know its uncompressed or TOC digest", info.digest.String()) } } - id := diffID.Hex() + id := diffIDOrTOCDigest.Hex() if lastLayer != "" { - id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() + id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffIDOrTOCDigest.Hex())).Hex() } if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { // There's already a layer that should have the right contents, just reuse it. lastLayer = layer.ID s.indexToStorageID[index] = &lastLayer - return nil + return false, nil } s.lock.Lock() diffOutput, ok := s.diffOutputs[info.digest] s.lock.Unlock() if ok { + if s.manifest == nil { + logrus.Debugf("Skipping commit for TOC=%q, manifest not yet available", id) + return true, nil + } + + man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest)) + if err != nil { + return false, fmt.Errorf("parsing manifest: %w", err) + } + + cb, err := s.getConfigBlob(man.ConfigInfo()) + if err != nil { + return false, err + } + + // retrieve the expected uncompressed digest from the config blob. + configOCI := &imgspecv1.Image{} + if err := json.Unmarshal(cb, configOCI); err != nil { + return false, err + } + if index >= len(configOCI.RootFS.DiffIDs) { + return false, fmt.Errorf("index %d out of range for configOCI.RootFS.DiffIDs", index) + } + layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil) if err != nil { - return err + return false, err } - // FIXME: what to do with the uncompressed digest? - diffOutput.UncompressedDigest = info.digest + // let the storage layer know what was the original uncompressed layer. + flags := make(map[string]interface{}) + flags[expectedLayerDiffIDFlag] = configOCI.RootFS.DiffIDs[index] + logrus.Debugf("Setting uncompressed digest to %q for layer %q", configOCI.RootFS.DiffIDs[index], id) + options := &graphdriver.ApplyDiffWithDifferOpts{ + Flags: flags, + } - if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil { + if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, options); err != nil { _ = s.imageRef.transport.store.Delete(layer.ID) - return err + return false, err } s.indexToStorageID[index] = &layer.ID - return nil + return false, nil } s.lock.Lock() @@ -627,11 +692,11 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si if ok { layer, err := al.PutAs(id, lastLayer, nil) if err != nil && !errors.Is(err, storage.ErrDuplicateID) { - return fmt.Errorf("failed to put layer from digest and labels: %w", err) + return false, fmt.Errorf("failed to put layer from digest and labels: %w", err) } lastLayer = layer.ID s.indexToStorageID[index] = &lastLayer - return nil + return false, nil } // Check if we previously cached a file with that blob's contents. If we didn't, @@ -642,7 +707,7 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si if !ok { // Try to find the layer with contents matching that blobsum. layer := "" - layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) + layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffIDOrTOCDigest) if err2 == nil && len(layers) > 0 { layer = layers[0].ID } else { @@ -652,7 +717,7 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si } } if layer == "" { - return fmt.Errorf("locating layer for blob %q: %w", info.digest, err2) + return false, fmt.Errorf("locating layer for blob %q: %w", info.digest, err2) } // Read the layer's contents. noCompression := archive.Uncompressed @@ -661,17 +726,17 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si } diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions) if err2 != nil { - return fmt.Errorf("reading layer %q for blob %q: %w", layer, info.digest, err2) + return false, fmt.Errorf("reading layer %q for blob %q: %w", layer, info.digest, err2) } // Copy the layer diff to a file. Diff() takes a lock that it holds // until the ReadCloser that it returns is closed, and PutLayer() wants // the same lock, so the diff can't just be directly streamed from one // to the other. filename = s.computeNextBlobCacheFile() - file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) + file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0o600) if err != nil { diff.Close() - return fmt.Errorf("creating temporary file %q: %w", filename, err) + return false, fmt.Errorf("creating temporary file %q: %w", filename, err) } // Copy the data to the file. // TODO: This can take quite some time, and should ideally be cancellable using @@ -680,7 +745,7 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si diff.Close() file.Close() if err != nil { - return fmt.Errorf("storing blob to file %q: %w", filename, err) + return false, fmt.Errorf("storing blob to file %q: %w", filename, err) } // Make sure that we can find this file later, should we need the layer's // contents again. @@ -691,21 +756,21 @@ func (s *storageImageDestination) commitLayer(index int, info addedLayerInfo, si // Read the cached blob and use it as a diff. file, err := os.Open(filename) if err != nil { - return fmt.Errorf("opening file %q: %w", filename, err) + return false, fmt.Errorf("opening file %q: %w", filename, err) } defer file.Close() // Build the new layer using the diff, regardless of where it came from. // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, &storage.LayerOptions{ OriginalDigest: info.digest, - UncompressedDigest: diffID, + UncompressedDigest: diffIDOrTOCDigest, }, file) if err != nil && !errors.Is(err, storage.ErrDuplicateID) { - return fmt.Errorf("adding layer with blob %q: %w", info.digest, err) + return false, fmt.Errorf("adding layer with blob %q: %w", info.digest, err) } s.indexToStorageID[index] = &layer.ID - return nil + return false, nil } // Commit marks the process of storing the image as successful and asks for the image to be persisted. @@ -752,11 +817,13 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t // Extract, commit, or find the layers. for i, blob := range layerBlobs { - if err := s.commitLayer(i, addedLayerInfo{ + if stopQueue, err := s.commitLayer(i, addedLayerInfo{ digest: blob.Digest, emptyLayer: blob.EmptyLayer, }, blob.Size); err != nil { return err + } else if stopQueue { + return fmt.Errorf("Internal error: storageImageDestination.Commit(): commitLayer() not ready to commit for layer %q", blob.Digest) } } var lastLayer string diff --git a/vendor/github.com/containers/image/v5/storage/storage_src.go b/vendor/github.com/containers/image/v5/storage/storage_src.go index f1ce0861e0c..28df60da7b1 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_src.go +++ b/vendor/github.com/containers/image/v5/storage/storage_src.go @@ -29,21 +29,33 @@ import ( "github.com/sirupsen/logrus" ) +// getBlobMutexProtected is a struct to hold the state of the getBlobMutex mutex. +type getBlobMutexProtected struct { + // digestToLayerID is a lookup map from the layer digest (either the uncompressed digest or the TOC digest) to the + // layer ID in the store. + digestToLayerID map[digest.Digest]string + + // layerPosition stores where we are in reading a blob's layers + layerPosition map[digest.Digest]int +} + type storageImageSource struct { impl.Compat impl.PropertyMethodsInitialize stubs.NoGetBlobAtInitialize - imageRef storageReference - image *storage.Image - systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files - layerPosition map[digest.Digest]int // Where we are in reading a blob's layers - cachedManifest []byte // A cached copy of the manifest, if already known, or nil - getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice - SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice + imageRef storageReference + image *storage.Image + systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files + cachedManifest []byte // A cached copy of the manifest, if already known, or nil + getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions (it guards layerPosition and digestToLayerID) + getBlobMutexProtected getBlobMutexProtected + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice } +const expectedLayerDiffIDFlag = "expected-layer-diffid" + // newImageSource sets up an image for reading. func newImageSource(sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) { // First, locate the image. @@ -62,9 +74,12 @@ func newImageSource(sys *types.SystemContext, imageRef storageReference) (*stora imageRef: imageRef, systemContext: sys, image: img, - layerPosition: make(map[digest.Digest]int), SignatureSizes: []int{}, SignaturesSizes: make(map[digest.Digest][]int), + getBlobMutexProtected: getBlobMutexProtected{ + digestToLayerID: make(map[digest.Digest]string), + layerPosition: make(map[digest.Digest]int), + }, } image.Compat = impl.AddCompat(image) if img.Metadata != "" { @@ -91,6 +106,7 @@ func (s *storageImageSource) Close() error { func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (rc io.ReadCloser, n int64, err error) { // We need a valid digest value. digest := info.Digest + err = digest.Validate() if err != nil { return nil, 0, err @@ -100,10 +116,24 @@ func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, c return io.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil } - // Check if the blob corresponds to a diff that was used to initialize any layers. Our - // callers should try to retrieve layers using their uncompressed digests, so no need to - // check if they're using one of the compressed digests, which we can't reproduce anyway. - layers, _ := s.imageRef.transport.store.LayersByUncompressedDigest(digest) + var layers []storage.Layer + + // If the digest was overriden by LayerInfosForCopy, then we need to use the TOC digest + // to retrieve it from the storage. + s.getBlobMutex.Lock() + layerID, found := s.getBlobMutexProtected.digestToLayerID[digest] + s.getBlobMutex.Unlock() + + if found { + if layer, err := s.imageRef.transport.store.Layer(layerID); err == nil { + layers = []storage.Layer{*layer} + } + } else { + // Check if the blob corresponds to a diff that was used to initialize any layers. Our + // callers should try to retrieve layers using their uncompressed digests, so no need to + // check if they're using one of the compressed digests, which we can't reproduce anyway. + layers, _ = s.imageRef.transport.store.LayersByUncompressedDigest(digest) + } // If it's not a layer, then it must be a data item. if len(layers) == 0 { @@ -174,8 +204,8 @@ func (s *storageImageSource) getBlobAndLayerID(digest digest.Digest, layers []st // which claim to have the same contents, that we actually do have multiple layers, otherwise we could // just go ahead and use the first one every time. s.getBlobMutex.Lock() - i := s.layerPosition[digest] - s.layerPosition[digest] = i + 1 + i := s.getBlobMutexProtected.layerPosition[digest] + s.getBlobMutexProtected.layerPosition[digest] = i + 1 s.getBlobMutex.Unlock() if len(layers) > 0 { layer = layers[i%len(layers)] @@ -267,14 +297,35 @@ func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDige if err != nil { return nil, fmt.Errorf("reading layer %q in image %q: %w", layerID, s.image.ID, err) } - if layer.UncompressedDigest == "" { - return nil, fmt.Errorf("uncompressed digest for layer %q is unknown", layerID) + if layer.UncompressedDigest == "" && layer.TOCDigest == "" { + return nil, fmt.Errorf("uncompressed digest and TOC digest for layer %q is unknown", layerID) } if layer.UncompressedSize < 0 { return nil, fmt.Errorf("uncompressed size for layer %q is unknown", layerID) } + + blobDigest := layer.UncompressedDigest + + if layer.TOCDigest != "" { + if layer.Flags == nil || layer.Flags[expectedLayerDiffIDFlag] == nil { + return nil, fmt.Errorf("TOC digest %q for layer %q is present but %q flag is not set", layer.TOCDigest, layerID, expectedLayerDiffIDFlag) + } + if expectedDigest, ok := layer.Flags[expectedLayerDiffIDFlag].(string); ok { + // if the layer is stored by its TOC, report the expected diffID as the layer Digest + // but store the TOC digest so we can later retrieve it from the storage. + blobDigest, err = digest.Parse(expectedDigest) + if err != nil { + return nil, fmt.Errorf("parsing expected diffID %q for layer %q: %w", expectedDigest, layerID, err) + } + } else { + return nil, fmt.Errorf("TOC digest %q for layer %q is present but %q flag is not a string", layer.TOCDigest, layerID, expectedLayerDiffIDFlag) + } + } + s.getBlobMutex.Lock() + s.getBlobMutexProtected.digestToLayerID[blobDigest] = layer.ID + s.getBlobMutex.Unlock() blobInfo := types.BlobInfo{ - Digest: layer.UncompressedDigest, + Digest: blobDigest, Size: layer.UncompressedSize, MediaType: uncompressedLayerType, } @@ -384,7 +435,7 @@ func (s *storageImageSource) getSize() (int64, error) { if err != nil { return -1, err } - if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 { + if (layer.TOCDigest == "" && layer.UncompressedDigest == "") || layer.UncompressedSize < 0 { return -1, fmt.Errorf("size for layer %q is unknown, failing getSize()", layerID) } sum += layer.UncompressedSize diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index ab32d652e7d..f71ee69325e 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -73,6 +73,13 @@ type ApplyDiffOpts struct { ForceMask *os.FileMode } +// ApplyDiffWithDifferOpts contains optional arguments for ApplyDiffWithDiffer methods. +type ApplyDiffWithDifferOpts struct { + ApplyDiffOpts + + Flags map[string]interface{} +} + // InitFunc initializes the storage driver. type InitFunc func(homedir string, options Options) (Driver, error) @@ -223,9 +230,9 @@ type DriverWithDiffer interface { Driver // ApplyDiffWithDiffer applies the changes using the callback function. // If id is empty, then a staging directory is created. The staging directory is guaranteed to be usable with ApplyDiffFromStagingDirectory. - ApplyDiffWithDiffer(id, parent string, options *ApplyDiffOpts, differ Differ) (output DriverWithDifferOutput, err error) + ApplyDiffWithDiffer(id, parent string, options *ApplyDiffWithDifferOpts, differ Differ) (output DriverWithDifferOutput, err error) // ApplyDiffFromStagingDirectory applies the changes using the specified staging directory. - ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *DriverWithDifferOutput, options *ApplyDiffOpts) error + ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *DriverWithDifferOutput, options *ApplyDiffWithDifferOpts) error // CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors CleanupStagingDirectory(stagingDirectory string) error // DifferTarget gets the location where files are stored for the layer. diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 04ecf871fd0..f1c742d0aca 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -2049,7 +2049,7 @@ func (d *Driver) useComposeFs() bool { } // ApplyDiff applies the changes in the new layer using the specified function -func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) { +func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffWithDifferOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) { var idMappings *idtools.IDMappings if options != nil { idMappings = options.Mappings @@ -2100,7 +2100,7 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App } // ApplyDiffFromStagingDirectory applies the changes using the specified staging directory. -func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *graphdriver.DriverWithDifferOutput, options *graphdriver.ApplyDiffOpts) error { +func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *graphdriver.DriverWithDifferOutput, options *graphdriver.ApplyDiffWithDifferOpts) error { if filepath.Dir(stagingDirectory) != d.getStagingDir() { return fmt.Errorf("%q is not a staging directory", stagingDirectory) } @@ -2125,8 +2125,6 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri return err } - diffOutput.UncompressedDigest = diffOutput.TOCDigest - return os.Rename(stagingDirectory, diffPath) } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index d105e73f6cd..130de5444fd 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -126,6 +126,13 @@ type Layer struct { // as a DiffID. UncompressedDigest digest.Digest `json:"diff-digest,omitempty"` + // TOCDigest represents the digest of the Table of Contents (TOC) of the blob. + // This digest is utilized when the UncompressedDigest is not + // validated during the partial image pull process, but the + // TOC itself is validated. + // It serves as an alternative reference under these specific conditions. + TOCDigest digest.Digest `json:"toc-digest,omitempty"` + // UncompressedSize is the length of the blob that was last passed to // ApplyDiff() or create(), after we decompressed it. If // UncompressedDigest is not set, this should be treated as if it were @@ -228,6 +235,10 @@ type roLayerStore interface { // specified uncompressed digest value recorded for them. LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + // LayersByTOCDigest returns a slice of the layers with the + // specified uncompressed digest value recorded for them. + LayersByTOCDigest(d digest.Digest) ([]Layer, error) + // Layers returns a slice of the known layers. Layers() ([]Layer, error) } @@ -296,13 +307,13 @@ type rwLayerStore interface { // ApplyDiffWithDiffer applies the changes through the differ callback function. // If to is the empty string, then a staging directory is created by the driver. - ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) + ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) // CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors CleanupStagingDirectory(stagingDirectory string) error // ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff. - ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error + ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error // DifferTarget gets the location where files are stored for the layer. DifferTarget(id string) (string, error) @@ -337,6 +348,7 @@ type layerStore struct { bymount map[string]*Layer bycompressedsum map[digest.Digest][]string byuncompressedsum map[digest.Digest][]string + bytocsum map[digest.Digest][]string layerspathsModified [numLayerLocationIndex]time.Time // FIXME: This field is only set when constructing layerStore, but locking rules of the driver @@ -366,6 +378,7 @@ func copyLayer(l *Layer) *Layer { CompressedSize: l.CompressedSize, UncompressedDigest: l.UncompressedDigest, UncompressedSize: l.UncompressedSize, + TOCDigest: l.TOCDigest, CompressionType: l.CompressionType, ReadOnly: l.ReadOnly, volatileStore: l.volatileStore, @@ -745,6 +758,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { names := make(map[string]*Layer) compressedsums := make(map[digest.Digest][]string) uncompressedsums := make(map[digest.Digest][]string) + tocsums := make(map[digest.Digest][]string) var errorToResolveBySaving error // == nil; if there are multiple errors, this is one of them. if r.lockfile.IsReadWrite() { selinux.ClearLabels() @@ -765,6 +779,9 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { if layer.UncompressedDigest != "" { uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID) } + if layer.TOCDigest != "" { + tocsums[layer.TOCDigest] = append(tocsums[layer.TOCDigest], layer.ID) + } if layer.MountLabel != "" { selinux.ReserveLabel(layer.MountLabel) } @@ -792,6 +809,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { r.byname = names r.bycompressedsum = compressedsums r.byuncompressedsum = uncompressedsums + r.bytocsum = tocsums // Load and merge information about which layers are mounted, and where. if r.lockfile.IsReadWrite() { @@ -1112,7 +1130,7 @@ func (r *layerStore) Size(name string) (int64, error) { // We use the presence of a non-empty digest as an indicator that the size value was intentionally set, and that // a zero value is not just present because it was never set to anything else (which can happen if the layer was // created by a version of this library that didn't keep track of digest and size information). - if layer.UncompressedDigest != "" { + if layer.TOCDigest != "" || layer.UncompressedDigest != "" { return layer.UncompressedSize, nil } return -1, nil @@ -1201,6 +1219,9 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s if layer.UncompressedDigest != "" { r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID) } + if layer.TOCDigest != "" { + r.bytocsum[layer.TOCDigest] = append(r.bytocsum[layer.TOCDigest], layer.ID) + } if err := r.saveFor(layer); err != nil { if e := r.Delete(layer.ID); e != nil { logrus.Errorf("While recovering from a failure to save layers, error deleting layer %#v: %v", id, e) @@ -1251,6 +1272,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount templateCompressedDigest digest.Digest templateCompressedSize int64 templateUncompressedDigest digest.Digest + templateTOCDigest digest.Digest templateUncompressedSize int64 templateCompressionType archive.Compression templateUIDs, templateGIDs []uint32 @@ -1263,6 +1285,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount } templateMetadata = templateLayer.Metadata templateIDMappings = idtools.NewIDMappingsFromMaps(templateLayer.UIDMap, templateLayer.GIDMap) + templateTOCDigest = templateLayer.TOCDigest templateCompressedDigest, templateCompressedSize = templateLayer.CompressedDigest, templateLayer.CompressedSize templateUncompressedDigest, templateUncompressedSize = templateLayer.UncompressedDigest, templateLayer.UncompressedSize templateCompressionType = templateLayer.CompressionType @@ -1291,6 +1314,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount CompressedDigest: templateCompressedDigest, CompressedSize: templateCompressedSize, UncompressedDigest: templateUncompressedDigest, + TOCDigest: templateTOCDigest, UncompressedSize: templateUncompressedSize, CompressionType: templateCompressionType, UIDs: templateUIDs, @@ -1413,6 +1437,9 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount if layer.UncompressedDigest != "" { r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID) } + if layer.TOCDigest != "" { + r.bytocsum[layer.TOCDigest] = append(r.bytocsum[layer.TOCDigest], layer.ID) + } } delete(layer.Flags, incompleteFlag) @@ -2197,6 +2224,25 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) { return r.driver.DiffSize(to, r.layerMappings(toLayer), from, r.layerMappings(fromLayer), toLayer.MountLabel) } +func updateDigestMap(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) { + var newList []string + if oldvalue != "" { + for _, value := range (*m)[oldvalue] { + if value != id { + newList = append(newList, value) + } + } + if len(newList) > 0 { + (*m)[oldvalue] = newList + } else { + delete(*m, oldvalue) + } + } + if newvalue != "" { + (*m)[newvalue] = append((*m)[newvalue], id) + } +} + // Requires startWriting. func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error) { return r.applyDiffWithOptions(to, nil, diff) @@ -2313,24 +2359,6 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions, uncompressedDigest = uncompressedDigester.Digest() } - updateDigestMap := func(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) { - var newList []string - if oldvalue != "" { - for _, value := range (*m)[oldvalue] { - if value != id { - newList = append(newList, value) - } - } - if len(newList) > 0 { - (*m)[oldvalue] = newList - } else { - delete(*m, oldvalue) - } - } - if newvalue != "" { - (*m)[newvalue] = append((*m)[newvalue], id) - } - } updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest, layer.ID) layer.CompressedDigest = compressedDigest layer.CompressedSize = compressedCounter.Count @@ -2372,7 +2400,7 @@ func (r *layerStore) DifferTarget(id string) (string, error) { } // Requires startWriting. -func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error { +func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error { ddriver, ok := r.driver.(drivers.DriverWithDiffer) if !ok { return ErrNotSupported @@ -2382,20 +2410,35 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, return ErrLayerUnknown } if options == nil { - options = &drivers.ApplyDiffOpts{ - Mappings: r.layerMappings(layer), - MountLabel: layer.MountLabel, + options = &drivers.ApplyDiffWithDifferOpts{ + ApplyDiffOpts: drivers.ApplyDiffOpts{ + Mappings: r.layerMappings(layer), + MountLabel: layer.MountLabel, + }, + Flags: nil, } } + err := ddriver.ApplyDiffFromStagingDirectory(layer.ID, layer.Parent, stagingDirectory, diffOutput, options) if err != nil { return err } layer.UIDs = diffOutput.UIDs layer.GIDs = diffOutput.GIDs + updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, diffOutput.UncompressedDigest, layer.ID) layer.UncompressedDigest = diffOutput.UncompressedDigest + updateDigestMap(&r.bytocsum, diffOutput.TOCDigest, diffOutput.TOCDigest, layer.ID) + layer.TOCDigest = diffOutput.TOCDigest layer.UncompressedSize = diffOutput.Size layer.Metadata = diffOutput.Metadata + if options != nil && options.Flags != nil { + if layer.Flags == nil { + layer.Flags = make(map[string]interface{}) + } + for k, v := range options.Flags { + layer.Flags[k] = v + } + } if len(diffOutput.TarSplit) != 0 { tsdata := bytes.Buffer{} compressor, err := pgzip.NewWriterLevel(&tsdata, pgzip.BestSpeed) @@ -2432,7 +2475,7 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, } // Requires startWriting. -func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { +func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { ddriver, ok := r.driver.(drivers.DriverWithDiffer) if !ok { return nil, ErrNotSupported @@ -2448,9 +2491,11 @@ func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOp return nil, ErrLayerUnknown } if options == nil { - options = &drivers.ApplyDiffOpts{ - Mappings: r.layerMappings(layer), - MountLabel: layer.MountLabel, + options = &drivers.ApplyDiffWithDifferOpts{ + ApplyDiffOpts: drivers.ApplyDiffOpts{ + Mappings: r.layerMappings(layer), + MountLabel: layer.MountLabel, + }, } } output, err := ddriver.ApplyDiffWithDiffer(layer.ID, layer.Parent, options, differ) @@ -2494,6 +2539,11 @@ func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error return r.layersByDigestMap(r.byuncompressedsum, d) } +// Requires startReading or startWriting. +func (r *layerStore) LayersByTOCDigest(d digest.Digest) ([]Layer, error) { + return r.layersByDigestMap(r.bytocsum, d) +} + func closeAll(closes ...func() error) (rErr error) { for _, f := range closes { if err := f(); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go b/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go new file mode 100644 index 00000000000..9cfd97d8eca --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go @@ -0,0 +1,34 @@ +package toc + +import ( + "github.com/containers/storage/pkg/chunked/internal" + digest "github.com/opencontainers/go-digest" +) + +// tocJSONDigestAnnotation is the annotation key for the digest of the estargz +// TOC JSON. +// It is defined in github.com/containerd/stargz-snapshotter/estargz as TOCJSONDigestAnnotation +// Duplicate it here to avoid a dependency on the package. +const tocJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest" + +// GetTOCDigest returns the digest of the TOC as recorded in the annotations. +// This function retrieves a digest that represents the content of a +// table of contents (TOC) from the image's annotations. +// This is an experimental feature and may be changed/removed in the future. +func GetTOCDigest(annotations map[string]string) (*digest.Digest, error) { + if contentDigest, ok := annotations[tocJSONDigestAnnotation]; ok { + d, err := digest.Parse(contentDigest) + if err != nil { + return nil, err + } + return &d, nil + } + if contentDigest, ok := annotations[internal.ManifestChecksumKey]; ok { + d, err := digest.Parse(contentDigest) + if err != nil { + return nil, err + } + return &d, nil + } + return nil, nil +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 62a9c9ea1b7..49a4ff1118b 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -315,10 +315,10 @@ type Store interface { // ApplyDiffer applies a diff to a layer. // It is the caller responsibility to clean the staging directory if it is not // successfully applied with ApplyDiffFromStagingDirectory. - ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) + ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) // ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff. - ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error + ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error // CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors CleanupStagingDirectory(stagingDirectory string) error @@ -334,6 +334,10 @@ type Store interface { // specified uncompressed digest value recorded for them. LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + // LayersByTOCDigest returns a slice of the layers with the + // specified TOC digest value recorded for them. + LayersByTOCDigest(d digest.Digest) ([]Layer, error) + // LayerSize returns a cached approximation of the layer's size, or -1 // if we don't have a value on hand. LayerSize(id string) (int64, error) @@ -2927,7 +2931,7 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro return nil, ErrLayerUnknown } -func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error { +func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error { _, err := writeToLayerStore(s, func(rlstore rwLayerStore) (struct{}, error) { if !rlstore.Exists(to) { return struct{}{}, ErrLayerUnknown @@ -2944,7 +2948,7 @@ func (s *store) CleanupStagingDirectory(stagingDirectory string) error { return err } -func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { +func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) { return writeToLayerStore(s, func(rlstore rwLayerStore) (*drivers.DriverWithDifferOutput, error) { if to != "" && !rlstore.Exists(to) { return nil, ErrLayerUnknown @@ -3006,6 +3010,13 @@ func (s *store) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d) } +func (s *store) LayersByTOCDigest(d digest.Digest) ([]Layer, error) { + if err := d.Validate(); err != nil { + return nil, fmt.Errorf("looking for TOC matching digest %q: %w", d, err) + } + return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByTOCDigest(d) }, d) +} + func (s *store) LayerSize(id string) (int64, error) { if res, done, err := readAllLayerStores(s, func(store roLayerStore) (int64, bool, error) { if store.Exists(id) { diff --git a/vendor/github.com/go-openapi/strfmt/time.go b/vendor/github.com/go-openapi/strfmt/time.go index 9bef4c3b335..682235dae5d 100644 --- a/vendor/github.com/go-openapi/strfmt/time.go +++ b/vendor/github.com/go-openapi/strfmt/time.go @@ -76,6 +76,8 @@ const ( ISO8601TimeWithReducedPrecisionLocaltime = "2006-01-02T15:04" // ISO8601TimeUniversalSortableDateTimePattern represents a ISO8601 universal sortable date time pattern. ISO8601TimeUniversalSortableDateTimePattern = "2006-01-02 15:04:05" + // short form of ISO8601TimeUniversalSortableDateTimePattern + ISO8601TimeUniversalSortableDateTimePatternShortForm = "2006-01-02" // DateTimePattern pattern to match for the date-time format from http://tools.ietf.org/html/rfc3339#section-5.6 DateTimePattern = `^([0-9]{2}):([0-9]{2}):([0-9]{2})(.[0-9]+)?(z|([+-][0-9]{2}:[0-9]{2}))$` ) @@ -84,7 +86,7 @@ var ( rxDateTime = regexp.MustCompile(DateTimePattern) // DateTimeFormats is the collection of formats used by ParseDateTime() - DateTimeFormats = []string{RFC3339Micro, RFC3339MicroNoColon, RFC3339Millis, RFC3339MillisNoColon, time.RFC3339, time.RFC3339Nano, ISO8601LocalTime, ISO8601TimeWithReducedPrecision, ISO8601TimeWithReducedPrecisionLocaltime, ISO8601TimeUniversalSortableDateTimePattern} + DateTimeFormats = []string{RFC3339Micro, RFC3339MicroNoColon, RFC3339Millis, RFC3339MillisNoColon, time.RFC3339, time.RFC3339Nano, ISO8601LocalTime, ISO8601TimeWithReducedPrecision, ISO8601TimeWithReducedPrecisionLocaltime, ISO8601TimeUniversalSortableDateTimePattern, ISO8601TimeUniversalSortableDateTimePatternShortForm} // MarshalFormat sets the time resolution format used for marshaling time (set to milliseconds) MarshalFormat = RFC3339Millis diff --git a/vendor/github.com/vbauerster/mpb/v8/README.md b/vendor/github.com/vbauerster/mpb/v8/README.md index 09825ca0861..af97c92a759 100644 --- a/vendor/github.com/vbauerster/mpb/v8/README.md +++ b/vendor/github.com/vbauerster/mpb/v8/README.md @@ -42,11 +42,9 @@ func main() { mpb.BarStyle().Lbound("╢").Filler("▌").Tip("▌").Padding("░").Rbound("╟"), mpb.PrependDecorators( // display our name with one space on the right - decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), + decor.Name(name, decor.WC{C: decor.DindentRight | decor.DextraSpace}), // replace ETA decorator with "done" message, OnComplete event - decor.OnComplete( - decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done", - ), + decor.OnComplete(decor.AverageETA(decor.ET_STYLE_GO), "done"), ), mpb.AppendDecorators(decor.Percentage()), ) diff --git a/vendor/github.com/vbauerster/mpb/v8/decor/decorator.go b/vendor/github.com/vbauerster/mpb/v8/decor/decorator.go index f537d3f7af4..31062ebd3a0 100644 --- a/vendor/github.com/vbauerster/mpb/v8/decor/decorator.go +++ b/vendor/github.com/vbauerster/mpb/v8/decor/decorator.go @@ -8,29 +8,27 @@ import ( ) const ( - // DidentRight bit specifies identation direction. + // DindentRight sets indentation from right to left. // - // |foo |b | With DidentRight - // | foo| b| Without DidentRight - DidentRight = 1 << iota + // |foo |b | DindentRight is set + // | foo| b| DindentRight is not set + DindentRight = 1 << iota - // DextraSpace bit adds extra space, makes sense with DSyncWidth only. - // When DidentRight bit set, the space will be added to the right, - // otherwise to the left. + // DextraSpace bit adds extra indentation space. DextraSpace // DSyncWidth bit enables same column width synchronization. // Effective with multiple bars only. DSyncWidth - // DSyncWidthR is shortcut for DSyncWidth|DidentRight - DSyncWidthR = DSyncWidth | DidentRight + // DSyncWidthR is shortcut for DSyncWidth|DindentRight + DSyncWidthR = DSyncWidth | DindentRight // DSyncSpace is shortcut for DSyncWidth|DextraSpace DSyncSpace = DSyncWidth | DextraSpace - // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight - DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight + // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DindentRight + DSyncSpaceR = DSyncWidth | DextraSpace | DindentRight ) // TimeStyle enum. @@ -143,11 +141,10 @@ func (wc WC) Format(str string) (string, int) { viewWidth := runewidth.StringWidth(str) if wc.W > viewWidth { viewWidth = wc.W + } else if (wc.C & DextraSpace) != 0 { + viewWidth++ } if (wc.C & DSyncWidth) != 0 { - if (wc.C & DextraSpace) != 0 { - viewWidth++ - } wc.wsync <- viewWidth viewWidth = <-wc.wsync } @@ -156,7 +153,7 @@ func (wc WC) Format(str string) (string, int) { // Init initializes width related config. func (wc *WC) Init() WC { - if (wc.C & DidentRight) != 0 { + if (wc.C & DindentRight) != 0 { wc.fill = runewidth.FillRight } else { wc.fill = runewidth.FillLeft diff --git a/vendor/modules.txt b/vendor/modules.txt index d8b996d276d..864d79a8ea1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -142,7 +142,7 @@ github.com/containers/common/pkg/umask github.com/containers/common/pkg/util github.com/containers/common/pkg/version github.com/containers/common/version -# github.com/containers/image/v5 v5.29.1-0.20231120202631-293b00ba7166 +# github.com/containers/image/v5 v5.29.1-0.20231206154545-28a299f3d39e ## explicit; go 1.19 github.com/containers/image/v5/copy github.com/containers/image/v5/directory @@ -233,7 +233,7 @@ github.com/containers/ocicrypt/keywrap/pkcs7 github.com/containers/ocicrypt/spec github.com/containers/ocicrypt/utils github.com/containers/ocicrypt/utils/keyprovider -# github.com/containers/storage v1.51.1-0.20231204015418-15c3cb7881e4 +# github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5 ## explicit; go 1.19 github.com/containers/storage github.com/containers/storage/drivers @@ -254,6 +254,7 @@ github.com/containers/storage/pkg/chunked github.com/containers/storage/pkg/chunked/compressor github.com/containers/storage/pkg/chunked/dump github.com/containers/storage/pkg/chunked/internal +github.com/containers/storage/pkg/chunked/toc github.com/containers/storage/pkg/config github.com/containers/storage/pkg/devicemapper github.com/containers/storage/pkg/directory @@ -389,7 +390,7 @@ github.com/go-openapi/runtime # github.com/go-openapi/spec v0.20.9 ## explicit; go 1.13 github.com/go-openapi/spec -# github.com/go-openapi/strfmt v0.21.7 +# github.com/go-openapi/strfmt v0.21.8 ## explicit; go 1.19 github.com/go-openapi/strfmt # github.com/go-openapi/swag v0.22.4 @@ -639,7 +640,7 @@ github.com/sigstore/fulcio/pkg/certificate # github.com/sigstore/rekor v1.2.2 ## explicit; go 1.19 github.com/sigstore/rekor/pkg/generated/models -# github.com/sigstore/sigstore v1.7.5 +# github.com/sigstore/sigstore v1.7.6 ## explicit; go 1.20 github.com/sigstore/sigstore/pkg/cryptoutils github.com/sigstore/sigstore/pkg/signature @@ -684,7 +685,7 @@ github.com/ulikunitz/xz/lzma github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage -# github.com/vbauerster/mpb/v8 v8.6.2 +# github.com/vbauerster/mpb/v8 v8.7.0 ## explicit; go 1.17 github.com/vbauerster/mpb/v8 github.com/vbauerster/mpb/v8/cwriter @@ -758,7 +759,7 @@ golang.org/x/exp/slices # golang.org/x/mod v0.13.0 ## explicit; go 1.18 golang.org/x/mod/semver -# golang.org/x/net v0.18.0 +# golang.org/x/net v0.19.0 ## explicit; go 1.18 golang.org/x/net/context golang.org/x/net/html