From cf38f611ea0f6bd7d8a7686230460825196d4160 Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Sat, 12 Nov 2022 00:16:52 -0500 Subject: [PATCH] fix: bugs found in integration test --- README.md | 10 +++-- client/client.go | 48 +++++++++++++++++----- client/client_test.go | 15 ------- client/fs/fs.go | 56 +++++++++++++++++++++++--- client/fs/instance.go | 3 +- client/manager.go | 12 +++--- client/snapshotter/plugin.go | 34 ++++++++-------- demo/deb-package/debian/control | 4 +- demo/deb-package/generate-changelog.sh | 2 +- docs/newbie.md | 2 +- docs/starlight-snapshotter.md | 2 +- util/receive/image.go | 24 ++++++++++- 12 files changed, 149 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 377e3f3..60a1ddb 100644 --- a/README.md +++ b/README.md @@ -168,8 +168,9 @@ Starlight is not complete. Our roadmap: |----------------------------------------------------------|-------------|-------------------| | [v0.1.3](https://github.com/mc256/starlight/tree/v0.1.3) | stable | | | [v0.2.0](https://github.com/mc256/starlight) | development | | -| v0.3.0 | | 2022-12-01 | -| v0.4.0 | | 2023-01-01 | +| v0.3.0 | | 2022-12-15 | +| v0.4.0 | | 2023-01-15 | +| v0.5.0 | | 2023-02-15 | Feature List: - [x] Scalable database backend (v0.2) @@ -186,4 +187,7 @@ Feature List: - [x] Multiple platforms image support (v0.2) - [ ] Goharbor Hook/ Scanner for automatic image conversion (v0.3) - [ ] Jointly optimizing multiple containers deployments (v0.4) - - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4) \ No newline at end of file + - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4) +- [ ] Starlight new features (v0.5) + - [ ] Resume interrupted pull connection (v0.4) + - [ ] Garbage Collection (v0.4) \ No newline at end of file diff --git a/client/client.go b/client/client.go index fb1388d..8ba2dd6 100644 --- a/client/client.go +++ b/client/client.go @@ -85,6 +85,9 @@ type Client struct { defaultOptimizeGroup string } +// ----------------------------------------------------------------------------- +// Base Image Searching + func escapeSlashes(s string) string { s = strings.ReplaceAll(s, "\\", "\\\\") return strings.ReplaceAll(s, "/", "\\/") @@ -92,9 +95,13 @@ func escapeSlashes(s string) string { func getImageFilter(ref string) string { return fmt.Sprintf( - "name~=/^%s.*/,labels.%s==%s", + "name~=/^%s.*/,labels.%s==%s,%s", + // choose images with the same name (just the tags are different) escapeSlashes(ref), + // choose images that are pulled by starlight util.ImageLabelPuller, "starlight", + // choose completed images + "labels."+util.ContentLabelCompletion, ) } @@ -102,9 +109,6 @@ func getDistributionSource(cfg string) string { return fmt.Sprintf("starlight.mc256.dev/distribution.source.%s", cfg) } -// ----------------------------------------------------------------------------- -// Base Image Searching - func (c *Client) findImage(filter string) (img containerd.Image, err error) { var list []containerd.Image list, err = c.client.ListImages(c.ctx, filter) @@ -126,11 +130,12 @@ func (c *Client) findImage(filter string) (img containerd.Image, err error) { nt = cur } } + // get the newest image return newest, nil } // FindBaseImage find the closest available image for the requested image, if user appointed an image, then this -// function will be used for looking up the appointed image +// function will be used for confirming the appointed image is available in the local storage func (c *Client) FindBaseImage(base, ref string) (img containerd.Image, err error) { var baseFilter string if base == "" { @@ -157,6 +162,7 @@ func (c *Client) FindBaseImage(base, ref string) (img containerd.Image, err erro // ----------------------------------------------------------------------------- // Image Pulling +// readBody is a helper function to read the body of a response and return it in a buffer func (c *Client) readBody(body io.ReadCloser, s int64) (*bytes.Buffer, error) { buf := bytes.NewBuffer(make([]byte, 0, s)) m, err := io.CopyN(buf, body, s) @@ -169,6 +175,11 @@ func (c *Client) readBody(body io.ReadCloser, s int64) (*bytes.Buffer, error) { return buf, nil } +// handleManifest unmarshal the manifest. +// It returns +// - the manifest in object +// - the manifest in bytes to store in the content store +// - error in case of failure func (c *Client) handleManifest(buf *bytes.Buffer) (manifest *v1.Manifest, b []byte, err error) { // decompress manifest r, err := gzip.NewReader(buf) @@ -187,6 +198,7 @@ func (c *Client) handleManifest(buf *bytes.Buffer) (manifest *v1.Manifest, b []b return manifest, man, nil } +// storeManifest saves the manifest in the content store with necessary labels func (c *Client) storeManifest(cfgName, d, ref, cfgd, sld string, man []byte) (err error) { pd := digest.Digest(d) @@ -195,11 +207,16 @@ func (c *Client) storeManifest(cfgName, d, ref, cfgd, sld string, man []byte) (e c.ctx, c.cs, pd.Hex(), bytes.NewReader(man), v1.Descriptor{Size: int64(len(man)), Digest: pd}, content.WithLabels(map[string]string{ - util.ImageLabelPuller: "starlight", - util.ContentLabelStarlightMediaType: "manifest", + // identifier + util.ImageLabelPuller: "starlight", + util.ContentLabelStarlightMediaType: "manifest", + + // garbage collection fmt.Sprintf("%s.config", util.ContentLabelContainerdGC): cfgd, fmt.Sprintf("%s.starlight", util.ContentLabelContainerdGC): sld, - getDistributionSource(cfgName): ref, + + // multiple starlight proxy support + getDistributionSource(cfgName): ref, })) if err != nil { return errors.Wrapf(err, "failed to open writer for manifest") @@ -207,7 +224,8 @@ func (c *Client) storeManifest(cfgName, d, ref, cfgd, sld string, man []byte) (e return nil } -func (c *Client) updateManifest(d string, chainIds []digest.Digest) (err error) { +// updateManifest marks the manifest as completed +func (c *Client) updateManifest(d string, chainIds []digest.Digest, t time.Time) (err error) { pd := digest.Digest(d) cs := c.client.ContentStore() @@ -218,7 +236,7 @@ func (c *Client) updateManifest(d string, chainIds []digest.Digest) (err error) return err } - info.Labels[util.ContentLabelCompletion] = time.Now().Format(time.RFC3339) + info.Labels[util.ContentLabelCompletion] = t.Format(time.RFC3339) // garbage collection tags, more info: // https://github.com/containerd/containerd/blob/83f44ddab5b17da74c5bd97dad7b2c5fa32871de/docs/garbage-collection.md @@ -524,8 +542,16 @@ func (c *Client) PullImage(base containerd.Image, ref, platform, proxyCfg string } // mark as completed + t := time.Now() + + // mark image as completed + ctrImg.Labels[util.ContentLabelCompletion] = t.Format(time.RFC3339) + if ctrImg, err = is.Update(c.ctx, ctrImg, "labels."+util.ContentLabelCompletion); err != nil { + return nil, errors.Wrapf(err, "failed to mark image as completed") + } + // update garbage collection labels - if err = c.updateManifest(md, chainIds); err != nil { + if err = c.updateManifest(md, chainIds, t); err != nil { return nil, errors.Wrapf(err, "failed to update manifest") } diff --git a/client/client_test.go b/client/client_test.go index d704457..1dc9870 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -248,21 +248,6 @@ func TestClient_LoadImage(t *testing.T) { t.Log(m) } -func TestClient_updateManifest(t *testing.T) { - cfg, _, _, _ := LoadConfig("/root/daemon.json") - c, err := NewClient(context.Background(), cfg) - if err != nil { - t.Error(err) - return - } - - err = c.updateManifest("sha256:50a0f37293a4d0880a49e0c41dd71e1d556d06d8fa6c8716afc467b1c7c52965") - if err != nil { - t.Error(err) - return - } -} - func TestPlatform(t *testing.T) { fmt.Println(platforms.DefaultString()) } diff --git a/client/fs/fs.go b/client/fs/fs.go index 42be8cd..7d5011c 100644 --- a/client/fs/fs.go +++ b/client/fs/fs.go @@ -7,6 +7,7 @@ package fs import ( "context" + "fmt" "github.com/containerd/containerd/log" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -32,6 +33,13 @@ type ReceivedFile interface { GetLinkName() string GetRealPath() string WaitForReady() + + // IsReferencingRequestedImage returns stack number where the actual content located + // if the file is available in the local filesystem then yes is false + IsReferencingRequestedImage() (stack int64, yes bool) + + // IsReferencingLocalFilesystem can not return true if IsReferencingRequestedImage returns true + IsReferencingLocalFilesystem() (serial int64, yes bool) } type StarlightFsNode struct { @@ -44,10 +52,19 @@ func (n *StarlightFsNode) getFile(p string) ReceivedFile { return n.instance.manager.LookUpFile(n.instance.stack, p) } -func (n *StarlightFsNode) getRealPath() string { - p := n.instance.manager.GetPathByLayer(n.instance.stack) +func (n *StarlightFsNode) getRealPath() (string, error) { + // 1. not available, in the same layer + // 2. not available, in other layers + // 3. available, in local filesystem pp := n.GetRealPath() - return filepath.Join(p, pp) + if stack, yes := n.ReceivedFile.IsReferencingRequestedImage(); yes { + return filepath.Join(n.instance.manager.GetPathByStack(stack), pp), nil + } + if serial, yes := n.ReceivedFile.IsReferencingLocalFilesystem(); yes { + return filepath.Join(n.instance.manager.GetPathBySerial(serial), pp), nil + } + + return "", fmt.Errorf("fsnode: unknown file reference [%s]", n.GetName()) } func (n *StarlightFsNode) log(filename string, access, complete time.Time) { @@ -162,7 +179,14 @@ func (n *StarlightFsNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) var _ = (fs.NodeOpener)((*StarlightFsNode)(nil)) func (n *StarlightFsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) { - r := n.getRealPath() + r, err := n.getRealPath() + if err != nil { + log.G(ctx).WithFields(logrus.Fields{ + "_s": n.instance.stack, + "_r": r, + }).Error("open") + return nil, 0, syscall.ENODATA + } access := time.Now() if !n.IsReady() { @@ -188,7 +212,29 @@ func (n *StarlightFsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle var _ = (fs.NodeFsyncer)((*StarlightFsNode)(nil)) func (n *StarlightFsNode) Fsync(ctx context.Context, f fs.FileHandle, flags uint32) syscall.Errno { - r := n.getRealPath() + r, err := n.getRealPath() + if err != nil { + log.G(ctx).WithFields(logrus.Fields{ + "_s": n.instance.stack, + "_r": r, + }).Error("fsync") + return syscall.ENODATA + } + + access := time.Now() + if !n.IsReady() { + n.WaitForReady() + } + complete := time.Now() + name := n.GetName() + n.log(name, access, complete) + + log.G(ctx).WithFields(logrus.Fields{ + "f": name, + "_s": n.instance.stack, + "_r": r, + }).Trace("fsync") + fd, err := syscall.Open(r, int(flags), 0) if err != nil { return fs.ToErrno(err) diff --git a/client/fs/instance.go b/client/fs/instance.go index b219d0b..ea9bff4 100644 --- a/client/fs/instance.go +++ b/client/fs/instance.go @@ -13,7 +13,8 @@ import ( ) type ImageManager interface { - GetPathByLayer(stack int64) string + GetPathByStack(stack int64) string + GetPathBySerial(stack int64) string LookUpFile(stack int64, filename string) ReceivedFile LogTrace(stack int64, filename string, access, complete time.Time) } diff --git a/client/manager.go b/client/manager.go index 901ca4a..00d043b 100644 --- a/client/manager.go +++ b/client/manager.go @@ -41,6 +41,8 @@ type Manager struct { cfg *Configuration //layers is a map from filesystem serial to receive.ImageLayer object + // - should it require files from layers that are not in this image, + // checkout layers using the serial number layers map[int64]*receive.ImageLayer //stackSerialMap is a map convert stack to filesystem serial (on the proxy side), @@ -75,12 +77,12 @@ func (m *Manager) ignoreStack(stack int64) bool { return m.completedStack[stack] } -func (m *Manager) getPathByStack(stack int64) string { - return m.layers[m.stackSerialMap[stack]].Local +func (m *Manager) GetPathByStack(stack int64) string { + return m.GetPathBySerial(m.stackSerialMap[stack]) } -func (m *Manager) GetPathByLayer(stack int64) string { - return m.getPathByStack(stack) +func (m *Manager) GetPathBySerial(serial int64) string { + return m.layers[serial].Local } func (m *Manager) LookUpFile(stack int64, filename string) fs.ReceivedFile { @@ -124,7 +126,7 @@ func (m *Manager) Extract(r *io.ReadCloser) error { } // regular extraction - p := m.getPathByStack(c.Stack) + p := m.GetPathByStack(c.Stack) if err := os.MkdirAll(filepath.Join(p, c.GetBaseDir()), 0755); err != nil { return errors.Wrapf(err, "failed to create directory %s", filepath.Join(p, c.GetBaseDir())) } diff --git a/client/snapshotter/plugin.go b/client/snapshotter/plugin.go index b34045d..df17b0c 100644 --- a/client/snapshotter/plugin.go +++ b/client/snapshotter/plugin.go @@ -102,7 +102,7 @@ func (s *Plugin) Stat(ctx context.Context, key string) (snapshots.Info, error) { log.G(s.ctx).WithFields(logrus.Fields{ "name": info.Name, - }).Debug("stat") + }).Debug("sn: stat") return info, nil } @@ -125,7 +125,7 @@ func (s *Plugin) Update(ctx context.Context, info snapshots.Info, fieldpaths ... log.G(s.ctx).WithFields(logrus.Fields{ "name": info.Name, "usage": fieldpaths, - }).Debug("updated") + }).Debug("sn: updated") return info, nil } @@ -151,7 +151,7 @@ func (s *Plugin) Usage(ctx context.Context, key string) (snapshots.Usage, error) log.G(s.ctx).WithFields(logrus.Fields{ "key": key, "usage": usage, - }).Debug("usage") + }).Debug("sn: usage") return usage, nil } @@ -176,7 +176,7 @@ func (s *Plugin) Mounts(ctx context.Context, key string) ([]mount.Mount, error) log.G(ctx).WithFields(logrus.Fields{ "key": key, "mnt": mnt, - }).Debug("mount") + }).Debug("sn: mount") return mnt, nil } @@ -194,7 +194,7 @@ func (s *Plugin) newSnapshot(ctx context.Context, key, parent string, readonly b "key": key, "parent": parent, "_readonly": readonly, - }).Debug("prepare") + }).Debug("sn: prepare") c, t, err := s.ms.TransactionContext(ctx, true) if err != nil { @@ -218,7 +218,7 @@ func (s *Plugin) newSnapshot(ctx context.Context, key, parent string, readonly b ss, err := storage.CreateSnapshot(c, kind, key, parent, opts...) if err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") + log.G(ctx).WithError(rerr).Warn("sn: failed to rollback transaction") } return nil, fmt.Errorf("failed to create snapshot: %w", err) } @@ -243,7 +243,7 @@ func (s *Plugin) newSnapshot(ctx context.Context, key, parent string, readonly b // mount snapshot mnt, err := s.mounts(c, ss.ID, &inf) if err != nil { - log.G(s.ctx).WithError(err).Error("mount failed") + log.G(s.ctx).WithError(err).Error("sn: mount failed") return nil, err } @@ -258,7 +258,7 @@ func (s *Plugin) newSnapshot(ctx context.Context, key, parent string, readonly b "_readonly": readonly, "id": ss.ID, "_starlight": usingSL, - }).Debug("prepared") + }).Debug("sn: prepared") return mnt, nil } @@ -283,7 +283,7 @@ func (s *Plugin) Commit(ctx context.Context, name, key string, opts ...snapshots snId, inf, _, err = storage.GetInfo(c, key) if err != nil { if err = t.Rollback(); err != nil { - log.G(ctx).WithError(err).Warn("failed to rollback transaction") + log.G(ctx).WithError(err).Warn("sn: failed to rollback transaction") } return err } @@ -301,7 +301,7 @@ func (s *Plugin) Commit(ctx context.Context, name, key string, opts ...snapshots usage, err = fs.DiskUsage(c, s.client.GetFilesystemPath(unId)) if err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") + log.G(ctx).WithError(rerr).Warn("sn: failed to rollback transaction") } return err } @@ -316,7 +316,7 @@ func (s *Plugin) Commit(ctx context.Context, name, key string, opts ...snapshots // Commit if _, err = storage.CommitActive(c, key, name, snapshots.Usage(usage), opts...); err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") + log.G(ctx).WithError(rerr).Warn("sn: failed to rollback transaction") } return fmt.Errorf("failed to commit snapshot: %w", err) } @@ -325,7 +325,7 @@ func (s *Plugin) Commit(ctx context.Context, name, key string, opts ...snapshots "name": name, "key": key, "usage": usage, - }).Debug("committed") + }).Debug("sn: committed") return t.Commit() } @@ -368,7 +368,7 @@ func (s *Plugin) Remove(ctx context.Context, key string) (err error) { if err != nil { log.G(s.ctx).WithError(err).WithFields(logrus.Fields{ "key": key, - }).Warn("failed to remove snapshot mounting") + }).Warn("sn: failed to remove snapshot mounting") return err } } else { @@ -376,7 +376,7 @@ func (s *Plugin) Remove(ctx context.Context, key string) (err error) { if err = os.RemoveAll(s.getMountingPoint(snId)); err != nil { log.G(s.ctx).WithError(err).WithFields(logrus.Fields{ "key": key, - }).Warn("failed to remove snapshot mounting") + }).Warn("sn: failed to remove snapshot mounting") return err } } @@ -386,7 +386,7 @@ func (s *Plugin) Remove(ctx context.Context, key string) (err error) { "id": info.Name, "parents": info.Parent, "_starlight": usingSL, - }).Debug("remove") + }).Debug("sn: remove") return nil } @@ -502,7 +502,7 @@ func (s *Plugin) mounts(ctx context.Context, ssId string, inf *snapshots.Info) ( }, }, nil } else { - return nil, fmt.Errorf("please use native overlayfs") + return nil, fmt.Errorf("sn: please use native overlayfs") } } @@ -525,7 +525,7 @@ func (s *Plugin) mounts(ctx context.Context, ssId string, inf *snapshots.Info) ( log.G(s.ctx). WithField("managers", mdsl). - Debug("mount: prepare manager") + Debug("sn: prepare manager") lower := make([]string, 0) for i := len(stack) - 1; i >= 0; i-- { diff --git a/demo/deb-package/debian/control b/demo/deb-package/debian/control index 434d6e2..67e74c2 100644 --- a/demo/deb-package/debian/control +++ b/demo/deb-package/debian/control @@ -1,4 +1,4 @@ -Source: starlight-snapshotter +Source: starlight Section: devel Priority: optional Maintainer: Junlin Chen @@ -6,7 +6,7 @@ Build-Depends: build-essential, dh-systemd (>= 1.5), debhelper (>=10) Standards-Version: 0.0.0 Homepage: https://github.com/mc256/starlight -Package: starlight-snapshotter +Package: starlight Architecture: amd64 Depends: containerd (>= 1.4.13), kernel (>= 5.15.0) Description: Starlight is an accelerator for provisioning container-based applications. diff --git a/demo/deb-package/generate-changelog.sh b/demo/deb-package/generate-changelog.sh index d0a6f68..0a23c57 100755 --- a/demo/deb-package/generate-changelog.sh +++ b/demo/deb-package/generate-changelog.sh @@ -5,7 +5,7 @@ function logentry() { local version=$2 local version_number=`echo $2 | sed 's/v//g'` local version_number=`[[ -z "$version_number" ]] && echo "0.0.0" || echo $version_number` - echo "starlight-snapshotter ($version_number) unstable; urgency=low" + echo "starlight ($version_number) unstable; urgency=low" echo git --no-pager log --format=" * %s" $previous${previous:+..}$version echo diff --git a/docs/newbie.md b/docs/newbie.md index 779e826..c76e6f0 100644 --- a/docs/newbie.md +++ b/docs/newbie.md @@ -202,7 +202,7 @@ cat < /dev/null [proxy_plugins] [proxy_plugins.starlight] type = "snapshot" - address = "/run/starlight-grpc/starlight-snapshotter.socket" + address = "/run/starlight/starlight-daemon.sock" EOT ``` diff --git a/docs/starlight-snapshotter.md b/docs/starlight-snapshotter.md index 453981f..2d2a5da 100644 --- a/docs/starlight-snapshotter.md +++ b/docs/starlight-snapshotter.md @@ -28,7 +28,7 @@ wget "https://github.com/mc256/starlight/releases/download/v${SL_VERSION}/starli sudo apt install -f "./starlight-snapshotter_${SL_VERSION}_$ARCH.deb" ``` -Update systemd service file `/lib/systemd/system/starlight-snapshotter.service`. +Update systemd service file `/lib/systemd/system/starlight.service`. - Change `STARLIGHT_PROXY` to the address of the Starlight Proxy. - remove `--plain-http` if the Starlight Proxy is behind a HTTPS reverse proxy. ``` diff --git a/util/receive/image.go b/util/receive/image.go index 54ad1e1..f0a28b3 100644 --- a/util/receive/image.go +++ b/util/receive/image.go @@ -69,7 +69,8 @@ type ReferencedFile struct { // (This is Serial not Stack) ReferenceFsId int64 `json:"R,omitempty"` - // if the file is not available on the client then ReferenceFsId is zero and ReferenceStack is non-zero, + // if the file is not available on the client but on other layers in the requested imageļ¼Œ + // then ReferenceFsId is zero and ReferenceStack is non-zero, // expecting the file content in the delta bundle body // (This is Stack not Serial) ReferenceStack int64 `json:"T,omitempty"` @@ -161,6 +162,27 @@ func (r *ReferencedFile) WaitForReady() { <-*r.Ready } +func (r *ReferencedFile) IsReferencingRequestedImage() (stack int64, yes bool) { + if r.ReferenceFsId != 0 { + return 0, false + } + + // in the payload + if r.ReferenceStack != 0 { + // different layer + return r.ReferenceStack, true + } + // same layer + return r.Stack, true +} + +func (r *ReferencedFile) IsReferencingLocalFilesystem() (serial int64, yes bool) { + if r.ReferenceFsId != 0 { + return r.ReferenceFsId, true + } + return 0, false +} + // ------------------------------------------ // used in extract from the delta bundle //