From dd7420482d99f61884eb9d550594599b980b2c91 Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Tue, 17 Dec 2024 10:41:38 -0500 Subject: [PATCH] Add LayerWriter for block CIMs This commit adds a layer writer that can be used for extracting an image layer tar into a Block CIM format. Existing forked CIM layer writer was renamed to a common base type `cimLayerWriter`. Forked CIM layer writer & Block CIM layer writer both now extend this common base type to write layers in that specific format. This commit also removes some code that used `time.Now()` as the default timestamps for some files that it creates within the layer CIM. These timestamps cause differences in the layer CIMs generated from the same layer tar. This change fixes that. Signed-off-by: Amit Barve --- internal/wclayer/cim/block_cim_writer.go | 135 +++++++++++++++++ internal/wclayer/cim/cim_writer_test.go | 55 +++++++ .../wclayer/cim/{LayerWriter.go => common.go} | 142 +++++++----------- internal/wclayer/cim/file_writer.go | 3 + internal/wclayer/cim/forked_cim_writer.go | 78 ++++++++++ internal/wclayer/cim/pending.go | 7 + internal/wclayer/cim/process.go | 17 +-- pkg/ociwclayer/cim/import.go | 46 +++++- 8 files changed, 375 insertions(+), 108 deletions(-) create mode 100644 internal/wclayer/cim/block_cim_writer.go create mode 100644 internal/wclayer/cim/cim_writer_test.go rename internal/wclayer/cim/{LayerWriter.go => common.go} (66%) create mode 100644 internal/wclayer/cim/forked_cim_writer.go diff --git a/internal/wclayer/cim/block_cim_writer.go b/internal/wclayer/cim/block_cim_writer.go new file mode 100644 index 0000000000..1e7da68c05 --- /dev/null +++ b/internal/wclayer/cim/block_cim_writer.go @@ -0,0 +1,135 @@ +//go:build windows + +package cim + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/cimfs" +) + +// A BlockCIMLayerWriter implements the CIMLayerWriter interface to allow writing +// container image layers in the blocked cim format. +type BlockCIMLayerWriter struct { + *cimLayerWriter + // the layer that we are writing + layer *cimfs.BlockCIM + // parent layers + parentLayers []*cimfs.BlockCIM + // added files maintains a map of all files that have been added to this layer + addedFiles map[string]struct{} +} + +var _ CIMLayerWriter = &BlockCIMLayerWriter{} + +// NewBlockCIMLayerWriter writes the layer files in the block CIM format. +func NewBlockCIMLayerWriter(ctx context.Context, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ *BlockCIMLayerWriter, err error) { + if !cimfs.IsBlockCimSupported() { + return nil, fmt.Errorf("BlockCIM not supported on this build") + } else if layer.Type != cimfs.BlockCIMTypeSingleFile { + // we only support writing single file CIMs for now because in layer + // writing process we still need to write some files (registry hives) + // outside the CIM. We currently use the parent directory of the CIM (i.e + // the parent directory of block path in this case) for this. This can't + // be reliably done with the block device CIM since the block path + // provided will be a volume path. However, once we get rid of hive rollup + // step during layer import we should be able to support block device + // CIMs. + return nil, ErrBlockCIMWriterNotSupported + } + + parentLayerPaths := make([]string, 0, len(parentLayers)) + for _, pl := range parentLayers { + if pl.Type != layer.Type { + return nil, ErrBlockCIMParentTypeMismatch + } + parentLayerPaths = append(parentLayerPaths, filepath.Dir(pl.BlockPath)) + } + + cim, err := cimfs.CreateBlockCIM(layer.BlockPath, layer.CimName, layer.Type) + if err != nil { + return nil, fmt.Errorf("error in creating a new cim: %w", err) + } + defer func() { + if err != nil { + cErr := cim.Close() + if cErr != nil { + log.G(ctx).WithError(err).Warnf("failed to close cim after error: %s", cErr) + } + } + }() + + // std file writer writes registry hives outside the CIM for 2 reasons. 1. We can + // merge the hives of this layer with the parent layer hives and then write the + // merged hives into the CIM. 2. When importing child layer of this layer, we + // have access to the merges hives of this layer. + sfw, err := newStdFileWriter(filepath.Dir(layer.BlockPath), parentLayerPaths) + if err != nil { + return nil, fmt.Errorf("error in creating new standard file writer: %w", err) + } + + return &BlockCIMLayerWriter{ + layer: layer, + parentLayers: parentLayers, + addedFiles: make(map[string]struct{}), + cimLayerWriter: &cimLayerWriter{ + ctx: ctx, + cimWriter: cim, + stdFileWriter: sfw, + layerPath: filepath.Dir(layer.BlockPath), + parentLayerPaths: parentLayerPaths, + }, + }, nil +} + +// Add adds a file to the layer with given metadata. +func (cw *BlockCIMLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error { + cw.addedFiles[name] = struct{}{} + return cw.cimLayerWriter.Add(name, fileInfo, fileSize, securityDescriptor, extendedAttributes, reparseData) +} + +// Remove removes a file that was present in a parent layer from the layer. +func (cw *BlockCIMLayerWriter) Remove(name string) error { + // set active write to nil so that we panic if layer tar is incorrectly formatted. + cw.activeWriter = nil + err := cw.cimWriter.AddTombstone(name) + if err != nil { + return fmt.Errorf("failed to remove file : %w", err) + } + return nil +} + +// AddLink adds a hard link to the layer. Note that the link added here is evaluated only +// at the CIM merge time. So an invalid link will not throw an error here. +func (cw *BlockCIMLayerWriter) AddLink(name string, target string) error { + // set active write to nil so that we panic if layer tar is incorrectly formatted. + cw.activeWriter = nil + + // when adding links to a block CIM, we need to know if the target file is present + // in this same block CIM or if it is coming from one of the parent layers. If the + // file is in the same CIM we add a standard hard link. If the file is not in the + // same CIM we add a special type of link called merged link. This merged link is + // resolved when all the individual block CIM layers are merged. In order to + // reliably know if the target is a part of the CIM or not, we wait until all + // files are added and then lookup the added entries in a map to make the + // decision. + pendingLinkOp := func(c *cimfs.CimFsWriter) error { + if _, ok := cw.addedFiles[target]; ok { + // target was added in this layer - add a normal link. Once a + // hardlink is added that hardlink also becomes a valid target for + // other links so include it in the map. + cw.addedFiles[name] = struct{}{} + return c.AddLink(target, name) + } else { + // target is from a parent layer - add a merged link + return c.AddMergedLink(target, name) + } + } + cw.pendingOps = append(cw.pendingOps, pendingCimOpFunc(pendingLinkOp)) + return nil + +} diff --git a/internal/wclayer/cim/cim_writer_test.go b/internal/wclayer/cim/cim_writer_test.go new file mode 100644 index 0000000000..0abc19da74 --- /dev/null +++ b/internal/wclayer/cim/cim_writer_test.go @@ -0,0 +1,55 @@ +//go:build windows + +package cim + +import ( + "context" + "errors" + "testing" + + "github.com/Microsoft/hcsshim/pkg/cimfs" +) + +func TestSingleFileWriterTypeMismatch(t *testing.T) { + if !cimfs.IsBlockCimSupported() { + t.Skipf("BlockCIM not supported") + } + + layer := &cimfs.BlockCIM{ + Type: cimfs.BlockCIMTypeSingleFile, + BlockPath: "", + CimName: "", + } + + parent := &cimfs.BlockCIM{ + Type: cimfs.BlockCIMTypeDevice, + BlockPath: "", + CimName: "", + } + + _, err := NewBlockCIMLayerWriter(context.TODO(), layer, []*cimfs.BlockCIM{parent}) + if !errors.Is(err, ErrBlockCIMParentTypeMismatch) { + t.Fatalf("expected error `%s`, got `%s`", ErrBlockCIMParentTypeMismatch, err) + } +} + +func TestSingleFileWriterInvalidBlockType(t *testing.T) { + if !cimfs.IsBlockCimSupported() { + t.Skipf("BlockCIM not supported") + } + + layer := &cimfs.BlockCIM{ + BlockPath: "", + CimName: "", + } + + parent := &cimfs.BlockCIM{ + BlockPath: "", + CimName: "", + } + + _, err := NewBlockCIMLayerWriter(context.TODO(), layer, []*cimfs.BlockCIM{parent}) + if !errors.Is(err, ErrBlockCIMWriterNotSupported) { + t.Fatalf("expected error `%s`, got `%s`", ErrBlockCIMWriterNotSupported, err) + } +} diff --git a/internal/wclayer/cim/LayerWriter.go b/internal/wclayer/cim/common.go similarity index 66% rename from internal/wclayer/cim/LayerWriter.go rename to internal/wclayer/cim/common.go index 9315971b64..391a5aaeda 100644 --- a/internal/wclayer/cim/LayerWriter.go +++ b/internal/wclayer/cim/common.go @@ -10,39 +10,14 @@ import ( "strings" "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/pkg/cimfs" - "go.opencensus.io/trace" ) -// A CimLayerWriter implements the wclayer.LayerWriter interface to allow writing container -// image layers in the cim format. -// A cim layer consist of cim files (which are usually stored in the `cim-layers` directory and -// some other files which are stored in the directory of that layer (i.e the `path` directory). -type CimLayerWriter struct { - ctx context.Context - s *trace.Span - // path to the layer (i.e layer's directory) as provided by the caller. - // Even if a layer is stored as a cim in the cim directory, some files associated - // with a layer are still stored in this path. - layerPath string - // parent layer paths - parentLayerPaths []string - // Handle to the layer cim - writes to the cim file - cimWriter *cimfs.CimFsWriter - // Handle to the writer for writing files in the local filesystem - stdFileWriter *stdFileWriter - // reference to currently active writer either cimWriter or stdFileWriter - activeWriter io.Writer - // denotes if this layer has the UtilityVM directory - hasUtilityVM bool - // some files are written outside the cim during initial import (via stdFileWriter) because we need to - // make some modifications to these files before writing them to the cim. The pendingOps slice - // maintains a list of such delayed modifications to the layer cim. These modifications are applied at - // the very end of layer import process. - pendingOps []pendingCimOp -} +var ( + ErrBlockCIMWriterNotSupported = fmt.Errorf("writing block device CIM isn't supported") + ErrBlockCIMParentTypeMismatch = fmt.Errorf("parent layer block CIM type doesn't match with extraction layer") +) type hive struct { name string @@ -60,6 +35,24 @@ var ( } ) +// CIMLayerWriter is an interface that supports writing a new container image layer to the +// CIM format +type CIMLayerWriter interface { + // Add adds a file to the layer with given metadata. + Add(string, *winio.FileBasicInfo, int64, []byte, []byte, []byte) error + // AddLink adds a hard link to the layer. The target must already have been added. + AddLink(string, string) error + // AddAlternateStream adds an alternate stream to a file + AddAlternateStream(string, uint64) error + // Remove removes a file that was present in a parent layer from the layer. + Remove(string) error + // Write writes data to the current file. The data must be in the format of a Win32 + // backup stream. + Write([]byte) (int, error) + // Close finishes the layer writing process and releases any resources. + Close(context.Context) error +} + func isDeltaOrBaseHive(path string) bool { for _, hv := range hives { if strings.EqualFold(path, filepath.Join(wclayer.HivesPath, hv.delta)) || @@ -79,8 +72,33 @@ func isStdFile(path string) bool { path == wclayer.BcdFilePath || path == wclayer.BootMgrFilePath) } +// cimLayerWriter is a base struct that is further extended by forked cim writer & blocked +// cim writer to provide full functionality of writing layers. +type cimLayerWriter struct { + ctx context.Context + // Handle to the layer cim - writes to the cim file + cimWriter *cimfs.CimFsWriter + // Handle to the writer for writing files in the local filesystem + stdFileWriter *stdFileWriter + // reference to currently active writer either cimWriter or stdFileWriter + activeWriter io.Writer + // denotes if this layer has the UtilityVM directory + hasUtilityVM bool + // path to the layer (i.e layer's directory) as provided by the caller. + // Even if a layer is stored as a cim in the cim directory, some files associated + // with a layer are still stored in this path. + layerPath string + // parent layer paths + parentLayerPaths []string + // some files are written outside the cim during initial import (via stdFileWriter) because we need to + // make some modifications to these files before writing them to the cim. The pendingOps slice + // maintains a list of such delayed modifications to the layer cim. These modifications are applied at + // the very end of layer import process. + pendingOps []pendingCimOp +} + // Add adds a file to the layer with given metadata. -func (cw *CimLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error { +func (cw *cimLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error { if name == wclayer.UtilityVMPath { cw.hasUtilityVM = true } @@ -108,7 +126,7 @@ func (cw *CimLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo, fileSi } // AddLink adds a hard link to the layer. The target must already have been added. -func (cw *CimLayerWriter) AddLink(name string, target string) error { +func (cw *cimLayerWriter) AddLink(name string, target string) error { // set active write to nil so that we panic if layer tar is incorrectly formatted. cw.activeWriter = nil if isStdFile(target) { @@ -130,7 +148,7 @@ func (cw *CimLayerWriter) AddLink(name string, target string) error { // AddAlternateStream creates another alternate stream at the given // path. Any writes made after this call will go to that stream. -func (cw *CimLayerWriter) AddAlternateStream(name string, size uint64) error { +func (cw *cimLayerWriter) AddAlternateStream(name string, size uint64) error { if isStdFile(name) { // As of now there is no known case of std file having multiple data streams. // If such a file is encountered our assumptions are wrong. Error out. @@ -144,21 +162,14 @@ func (cw *CimLayerWriter) AddAlternateStream(name string, size uint64) error { return nil } -// Remove removes a file that was present in a parent layer from the layer. -func (cw *CimLayerWriter) Remove(name string) error { - // set active write to nil so that we panic if layer tar is incorrectly formatted. - cw.activeWriter = nil - return cw.cimWriter.Unlink(name) -} - // Write writes data to the current file. The data must be in the format of a Win32 // backup stream. -func (cw *CimLayerWriter) Write(b []byte) (int, error) { +func (cw *cimLayerWriter) Write(b []byte) (int, error) { return cw.activeWriter.Write(b) } // Close finishes the layer writing process and releases any resources. -func (cw *CimLayerWriter) Close(ctx context.Context) (retErr error) { +func (cw *cimLayerWriter) Close(ctx context.Context) (retErr error) { if err := cw.stdFileWriter.Close(ctx); err != nil { return err } @@ -170,7 +181,7 @@ func (cw *CimLayerWriter) Close(ctx context.Context) (retErr error) { } }() - // UVM based containers aren't supported with CimFS, don't process the UVM layer + // We don't support running UtilityVM with CIM layers yet. processUtilityVM := false if len(cw.parentLayerPaths) == 0 { @@ -190,50 +201,3 @@ func (cw *CimLayerWriter) Close(ctx context.Context) (retErr error) { } return nil } - -func NewCimLayerWriter(ctx context.Context, layerPath, cimPath string, parentLayerPaths, parentLayerCimPaths []string) (_ *CimLayerWriter, err error) { - if !cimfs.IsCimFSSupported() { - return nil, fmt.Errorf("CimFs not supported on this build") - } - - ctx, span := trace.StartSpan(ctx, "hcsshim::NewCimLayerWriter") - defer func() { - if err != nil { - oc.SetSpanStatus(span, err) - span.End() - } - }() - span.AddAttributes( - trace.StringAttribute("path", layerPath), - trace.StringAttribute("cimPath", cimPath), - trace.StringAttribute("parentLayerPaths", strings.Join(parentLayerCimPaths, ", ")), - trace.StringAttribute("parentLayerPaths", strings.Join(parentLayerPaths, ", "))) - - parentCim := "" - if len(parentLayerPaths) > 0 { - if filepath.Dir(cimPath) != filepath.Dir(parentLayerCimPaths[0]) { - return nil, fmt.Errorf("parent cim can not be stored in different directory") - } - // We only need to provide parent CIM name, it is assumed that both parent CIM - // and newly created CIM are present in the same directory. - parentCim = filepath.Base(parentLayerCimPaths[0]) - } - - cim, err := cimfs.Create(filepath.Dir(cimPath), parentCim, filepath.Base(cimPath)) - if err != nil { - return nil, fmt.Errorf("error in creating a new cim: %w", err) - } - - sfw, err := newStdFileWriter(layerPath, parentLayerPaths) - if err != nil { - return nil, fmt.Errorf("error in creating new standard file writer: %w", err) - } - return &CimLayerWriter{ - ctx: ctx, - s: span, - layerPath: layerPath, - parentLayerPaths: parentLayerPaths, - cimWriter: cim, - stdFileWriter: sfw, - }, nil -} diff --git a/internal/wclayer/cim/file_writer.go b/internal/wclayer/cim/file_writer.go index 497bbbbb9a..9e5e8dd456 100644 --- a/internal/wclayer/cim/file_writer.go +++ b/internal/wclayer/cim/file_writer.go @@ -86,5 +86,8 @@ func (sfw *stdFileWriter) Close(ctx context.Context) error { if err := sfw.closeActiveFile(); err != nil { return fmt.Errorf("failed to close active file %s : %w", sfw.activeFile.Name(), err) } + if err := sfw.root.Close(); err != nil { + return fmt.Errorf("failed to close root dir: %w", err) + } return nil } diff --git a/internal/wclayer/cim/forked_cim_writer.go b/internal/wclayer/cim/forked_cim_writer.go new file mode 100644 index 0000000000..7da052b515 --- /dev/null +++ b/internal/wclayer/cim/forked_cim_writer.go @@ -0,0 +1,78 @@ +//go:build windows + +package cim + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/cimfs" +) + +// A ForkedCimLayerWriter implements the wclayer.LayerWriter interface to allow writing container +// image layers in the cim format. +// A cim layer consist of cim files (which are usually stored in the `cim-layers` directory and +// some other files which are stored in the directory of that layer (i.e the `path` directory). +type ForkedCimLayerWriter struct { + *cimLayerWriter +} + +var _ CIMLayerWriter = &ForkedCimLayerWriter{} + +func NewForkedCimLayerWriter(ctx context.Context, layerPath, cimPath string, parentLayerPaths, parentLayerCimPaths []string) (_ *ForkedCimLayerWriter, err error) { + if !cimfs.IsCimFSSupported() { + return nil, fmt.Errorf("CimFs not supported on this build") + } + + parentCim := "" + if len(parentLayerPaths) > 0 { + // We only need to provide parent CIM name, it is assumed that both parent CIM + // and newly created CIM are present in the same directory. + parentCim = filepath.Base(parentLayerCimPaths[0]) + } + + cim, err := cimfs.Create(filepath.Dir(cimPath), parentCim, filepath.Base(cimPath)) + if err != nil { + return nil, fmt.Errorf("error in creating a new cim: %w", err) + } + defer func() { + if err != nil { + cErr := cim.Close() + if cErr != nil { + log.G(ctx).WithError(err).Warnf("failed to close cim after error: %s", cErr) + } + cErr = cimfs.DestroyCim(ctx, cimPath) + if cErr != nil { + log.G(ctx).WithError(err).Warnf("failed to cleanup cim after error: %s", cErr) + } + } + }() + + sfw, err := newStdFileWriter(layerPath, parentLayerPaths) + if err != nil { + return nil, fmt.Errorf("error in creating new standard file writer: %w", err) + } + return &ForkedCimLayerWriter{ + cimLayerWriter: &cimLayerWriter{ + parentLayerPaths: parentLayerPaths, + ctx: ctx, + cimWriter: cim, + stdFileWriter: sfw, + layerPath: layerPath, + }, + }, nil +} + +// Remove removes a file that was present in a parent layer from the layer. +func (cw *ForkedCimLayerWriter) Remove(name string) error { + // set active write to nil so that we panic if layer tar is incorrectly formatted. + cw.activeWriter = nil + err := cw.cimWriter.Unlink(name) + if err == nil || os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to remove file: %w", err) +} diff --git a/internal/wclayer/cim/pending.go b/internal/wclayer/cim/pending.go index d13bdff850..f2185a0998 100644 --- a/internal/wclayer/cim/pending.go +++ b/internal/wclayer/cim/pending.go @@ -16,6 +16,13 @@ type pendingCimOp interface { apply(cw *cimfs.CimFsWriter) error } +type pendingCimOpFunc func(cw *cimfs.CimFsWriter) error + +func (f pendingCimOpFunc) apply(cw *cimfs.CimFsWriter) error { + return f(cw) + +} + // add op represents a pending operation of adding a new file inside the cim type addOp struct { // path inside the cim at which the file should be added diff --git a/internal/wclayer/cim/process.go b/internal/wclayer/cim/process.go index 8fdb3bad3f..ace81122bc 100644 --- a/internal/wclayer/cim/process.go +++ b/internal/wclayer/cim/process.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/wclayer" @@ -34,10 +33,6 @@ func processBaseLayerHives(layerPath string) ([]pendingCimOp, error) { } hivesDirInfo := &winio.FileBasicInfo{ - CreationTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastAccessTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastWriteTime: windows.NsecToFiletime(time.Now().UnixNano()), - ChangeTime: windows.NsecToFiletime(time.Now().UnixNano()), FileAttributes: windows.FILE_ATTRIBUTE_DIRECTORY, } pendingOps = append(pendingOps, &addOp{ @@ -71,10 +66,6 @@ func processLayoutFile(layerPath string) ([]pendingCimOp, error) { } layoutFileInfo := &winio.FileBasicInfo{ - CreationTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastAccessTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastWriteTime: windows.NsecToFiletime(time.Now().UnixNano()), - ChangeTime: windows.NsecToFiletime(time.Now().UnixNano()), FileAttributes: windows.FILE_ATTRIBUTE_NORMAL, } @@ -89,7 +80,7 @@ func processLayoutFile(layerPath string) ([]pendingCimOp, error) { // Some of the layer files that are generated during the processBaseLayer call must be added back // inside the cim, some registry file links must be updated. This function takes care of all those // steps. This function opens the cim file for writing and updates it. -func (cw *CimLayerWriter) processBaseLayer(ctx context.Context, processUtilityVM bool) (err error) { +func (cw *cimLayerWriter) processBaseLayer(ctx context.Context, processUtilityVM bool) (err error) { if processUtilityVM { if err = processUtilityVMLayer(ctx, cw.layerPath); err != nil { return fmt.Errorf("process utilityVM layer: %w", err) @@ -113,7 +104,7 @@ func (cw *CimLayerWriter) processBaseLayer(ctx context.Context, processUtilityVM // processNonBaseLayer takes care of the processing required for a non base layer. As of now // the only processing required for non base layer is to merge the delta registry hives of the // non-base layer with it's parent layer. -func (cw *CimLayerWriter) processNonBaseLayer(ctx context.Context, processUtilityVM bool) (err error) { +func (cw *cimLayerWriter) processNonBaseLayer(ctx context.Context, processUtilityVM bool) (err error) { for _, hv := range hives { baseHive := filepath.Join(wclayer.HivesPath, hv.base) deltaHive := filepath.Join(wclayer.HivesPath, hv.delta) @@ -134,10 +125,6 @@ func (cw *CimLayerWriter) processNonBaseLayer(ctx context.Context, processUtilit pathInCim: baseHive, hostPath: filepath.Join(cw.layerPath, baseHive), fileInfo: &winio.FileBasicInfo{ - CreationTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastAccessTime: windows.NsecToFiletime(time.Now().UnixNano()), - LastWriteTime: windows.NsecToFiletime(time.Now().UnixNano()), - ChangeTime: windows.NsecToFiletime(time.Now().UnixNano()), FileAttributes: windows.FILE_ATTRIBUTE_NORMAL, }, }) diff --git a/pkg/ociwclayer/cim/import.go b/pkg/ociwclayer/cim/import.go index d8f4a1aa95..83d7a82cdf 100644 --- a/pkg/ociwclayer/cim/import.go +++ b/pkg/ociwclayer/cim/import.go @@ -18,7 +18,9 @@ import ( "github.com/Microsoft/go-winio/backuptar" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/wclayer/cim" + "github.com/Microsoft/hcsshim/pkg/cimfs" "github.com/Microsoft/hcsshim/pkg/ociwclayer" + "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -30,13 +32,20 @@ import ( // `parentLayerPaths` are paths to the parent layer directories. Ordered from highest to lowest. // // This function returns the total size of the layer's files, in bytes. -func ImportCimLayerFromTar(ctx context.Context, r io.Reader, layerPath, cimPath string, parentLayerPaths, parentLayerCimPaths []string) (int64, error) { - err := os.MkdirAll(layerPath, 0) +func ImportCimLayerFromTar(ctx context.Context, r io.Reader, layerPath, cimPath string, parentLayerPaths, parentLayerCimPaths []string) (_ int64, err error) { + log.G(ctx).WithFields(logrus.Fields{ + "layer path": layerPath, + "layer cim path": cimPath, + "parent layer paths": strings.Join(parentLayerPaths, ", "), + "parent layer CIM paths": strings.Join(parentLayerCimPaths, ", "), + }).Debug("Importing cim layer from tar") + + err = os.MkdirAll(layerPath, 0) if err != nil { return 0, err } - w, err := cim.NewCimLayerWriter(ctx, layerPath, cimPath, parentLayerPaths, parentLayerCimPaths) + w, err := cim.NewForkedCimLayerWriter(ctx, layerPath, cimPath, parentLayerPaths, parentLayerCimPaths) if err != nil { return 0, err } @@ -52,7 +61,36 @@ func ImportCimLayerFromTar(ctx context.Context, r io.Reader, layerPath, cimPath return n, nil } -func writeCimLayerFromTar(ctx context.Context, r io.Reader, w *cim.CimLayerWriter) (int64, error) { +// ImportSingleFileCimLayerFromTar reads a layer from an OCI layer tar stream and extracts +// it into the SingleFileCIM format. +func ImportSingleFileCimLayerFromTar(ctx context.Context, r io.Reader, layer *cimfs.BlockCIM, parentLayers []*cimfs.BlockCIM) (_ int64, err error) { + log.G(ctx).WithFields(logrus.Fields{ + "layer": layer, + "parent layers": fmt.Sprintf("%v", parentLayers), + }).Debug("Importing single file cim layer from tar") + + err = os.MkdirAll(filepath.Dir(layer.BlockPath), 0) + if err != nil { + return 0, err + } + + w, err := cim.NewBlockCIMLayerWriter(ctx, layer, parentLayers) + if err != nil { + return 0, err + } + + n, err := writeCimLayerFromTar(ctx, r, w) + cerr := w.Close(ctx) + if err != nil { + return 0, err + } + if cerr != nil { + return 0, cerr + } + return n, nil +} + +func writeCimLayerFromTar(ctx context.Context, r io.Reader, w cim.CIMLayerWriter) (int64, error) { tr := tar.NewReader(r) buf := bufio.NewWriter(w) size := int64(0)