fix: bugs found in integration test

README.md

@@ -168,8 +168,9 @@ Starlight is not complete. Our roadmap:
 |----------------------------------------------------------|-------------|-------------------|
 | [v0.1.3](https://github.com/mc256/starlight/tree/v0.1.3) | stable      |                   |
 | [v0.2.0](https://github.com/mc256/starlight)             | development |                   |
-| v0.3.0                                                   |             | 2022-12-01        |
-| v0.4.0                                                   |             | 2023-01-01        |
+| v0.3.0                                                   |             | 2022-12-15        |
+| v0.4.0                                                   |             | 2023-01-15        |
+| v0.5.0                                                   |             | 2023-02-15        |
 
 Feature List:
 - [x] Scalable database backend (v0.2)
@@ -186,4 +187,7 @@ Feature List:
   - [x] Multiple platforms image support (v0.2) 
   - [ ] Goharbor Hook/ Scanner for automatic image conversion (v0.3)
   - [ ] Jointly optimizing multiple containers deployments (v0.4)
-  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4)
+  - [ ] Converting containers that have already been fully retrieved using Starlight to use OverlayFS. (v0.4)
+- [ ] Starlight new features (v0.5)
+  - [ ] Resume interrupted pull connection (v0.4)
+  - [ ] Garbage Collection (v0.4)

client/client.go

@@ -85,26 +85,30 @@ type Client struct {
 	defaultOptimizeGroup string
 }
 
+// -----------------------------------------------------------------------------
+// Base Image Searching
+
 func escapeSlashes(s string) string {
 	s = strings.ReplaceAll(s, "\\", "\\\\")
 	return strings.ReplaceAll(s, "/", "\\/")
 }
 
 func getImageFilter(ref string) string {
 	return fmt.Sprintf(
-		"name~=/^%s.*/,labels.%s==%s",
+		"name~=/^%s.*/,labels.%s==%s,%s",
+		// choose images with the same name (just the tags are different)
 		escapeSlashes(ref),
+		// choose images that are pulled by starlight
 		util.ImageLabelPuller, "starlight",
+		// choose completed images
+		"labels."+util.ContentLabelCompletion,
 	)
 }
 
 func getDistributionSource(cfg string) string {
 	return fmt.Sprintf("starlight.mc256.dev/distribution.source.%s", cfg)
 }
 
-// -----------------------------------------------------------------------------
-// Base Image Searching
-
 func (c *Client) findImage(filter string) (img containerd.Image, err error) {
 	var list []containerd.Image
 	list, err = c.client.ListImages(c.ctx, filter)
@@ -126,11 +130,12 @@ func (c *Client) findImage(filter string) (img containerd.Image, err error) {
 			nt = cur
 		}
 	}
+	// get the newest image
 	return newest, nil
 }
 
 // FindBaseImage find the closest available image for the requested image, if user appointed an image, then this
-// function will be used for looking up the appointed image
+// function will be used for confirming the appointed image is available in the local storage
 func (c *Client) FindBaseImage(base, ref string) (img containerd.Image, err error) {
 	var baseFilter string
 	if base == "" {
@@ -157,6 +162,7 @@ func (c *Client) FindBaseImage(base, ref string) (img containerd.Image, err erro
 // -----------------------------------------------------------------------------
 // Image Pulling
 
+// readBody is a helper function to read the body of a response and return it in a buffer
 func (c *Client) readBody(body io.ReadCloser, s int64) (*bytes.Buffer, error) {
 	buf := bytes.NewBuffer(make([]byte, 0, s))
 	m, err := io.CopyN(buf, body, s)
@@ -169,6 +175,11 @@ func (c *Client) readBody(body io.ReadCloser, s int64) (*bytes.Buffer, error) {
 	return buf, nil
 }
 
+// handleManifest unmarshal the manifest.
+// It returns
+//  - the manifest in object
+//  - the manifest in bytes to store in the content store
+//  - error in case of failure
 func (c *Client) handleManifest(buf *bytes.Buffer) (manifest *v1.Manifest, b []byte, err error) {
 	// decompress manifest
 	r, err := gzip.NewReader(buf)
@@ -187,6 +198,7 @@ func (c *Client) handleManifest(buf *bytes.Buffer) (manifest *v1.Manifest, b []b
 	return manifest, man, nil
 }
 
+// storeManifest saves the manifest in the content store with necessary labels
 func (c *Client) storeManifest(cfgName, d, ref, cfgd, sld string, man []byte) (err error) {
 	pd := digest.Digest(d)
 
@@ -195,19 +207,25 @@ func (c *Client) storeManifest(cfgName, d, ref, cfgd, sld string, man []byte) (e
 		c.ctx, c.cs, pd.Hex(), bytes.NewReader(man),
 		v1.Descriptor{Size: int64(len(man)), Digest: pd},
 		content.WithLabels(map[string]string{
-			util.ImageLabelPuller:                                      "starlight",
-			util.ContentLabelStarlightMediaType:                        "manifest",
+			// identifier
+			util.ImageLabelPuller:               "starlight",
+			util.ContentLabelStarlightMediaType: "manifest",
+
+			// garbage collection
 			fmt.Sprintf("%s.config", util.ContentLabelContainerdGC):    cfgd,
 			fmt.Sprintf("%s.starlight", util.ContentLabelContainerdGC): sld,
-			getDistributionSource(cfgName):                             ref,
+
+			// multiple starlight proxy support
+			getDistributionSource(cfgName): ref,
 		}))
 	if err != nil {
 		return errors.Wrapf(err, "failed to open writer for manifest")
 	}
 	return nil
 }
 
-func (c *Client) updateManifest(d string, chainIds []digest.Digest) (err error) {
+// updateManifest marks the manifest as completed
+func (c *Client) updateManifest(d string, chainIds []digest.Digest, t time.Time) (err error) {
 	pd := digest.Digest(d)
 	cs := c.client.ContentStore()
 
@@ -218,7 +236,7 @@ func (c *Client) updateManifest(d string, chainIds []digest.Digest) (err error)
 		return err
 	}
 
-	info.Labels[util.ContentLabelCompletion] = time.Now().Format(time.RFC3339)
+	info.Labels[util.ContentLabelCompletion] = t.Format(time.RFC3339)
 
 	// garbage collection tags, more info:
 	// https://github.com/containerd/containerd/blob/83f44ddab5b17da74c5bd97dad7b2c5fa32871de/docs/garbage-collection.md
@@ -524,8 +542,16 @@ func (c *Client) PullImage(base containerd.Image, ref, platform, proxyCfg string
 	}
 
 	// mark as completed
+	t := time.Now()
+
+	// mark image as completed
+	ctrImg.Labels[util.ContentLabelCompletion] = t.Format(time.RFC3339)
+	if ctrImg, err = is.Update(c.ctx, ctrImg, "labels."+util.ContentLabelCompletion); err != nil {
+		return nil, errors.Wrapf(err, "failed to mark image as completed")
+	}
+
 	// update garbage collection labels
-	if err = c.updateManifest(md, chainIds); err != nil {
+	if err = c.updateManifest(md, chainIds, t); err != nil {
 		return nil, errors.Wrapf(err, "failed to update manifest")
 	}
 

client/client_test.go

@@ -248,21 +248,6 @@ func TestClient_LoadImage(t *testing.T) {
 	t.Log(m)
 }
 
-func TestClient_updateManifest(t *testing.T) {
-	cfg, _, _, _ := LoadConfig("/root/daemon.json")
-	c, err := NewClient(context.Background(), cfg)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	err = c.updateManifest("sha256:50a0f37293a4d0880a49e0c41dd71e1d556d06d8fa6c8716afc467b1c7c52965")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-}
-
 func TestPlatform(t *testing.T) {
 	fmt.Println(platforms.DefaultString())
 }

client/fs/fs.go

@@ -7,6 +7,7 @@ package fs
 
 import (
 	"context"
+	"fmt"
 	"github.com/containerd/containerd/log"
 	"github.com/hanwen/go-fuse/v2/fs"
 	"github.com/hanwen/go-fuse/v2/fuse"
@@ -32,6 +33,13 @@ type ReceivedFile interface {
 	GetLinkName() string
 	GetRealPath() string
 	WaitForReady()
+
+	// IsReferencingRequestedImage returns stack number where the actual content located
+	// if the file is available in the local filesystem then yes is false
+	IsReferencingRequestedImage() (stack int64, yes bool)
+
+	// IsReferencingLocalFilesystem can not return true if IsReferencingRequestedImage returns true
+	IsReferencingLocalFilesystem() (serial int64, yes bool)
 }
 
 type StarlightFsNode struct {
@@ -44,10 +52,19 @@ func (n *StarlightFsNode) getFile(p string) ReceivedFile {
 	return n.instance.manager.LookUpFile(n.instance.stack, p)
 }
 
-func (n *StarlightFsNode) getRealPath() string {
-	p := n.instance.manager.GetPathByLayer(n.instance.stack)
+func (n *StarlightFsNode) getRealPath() (string, error) {
+	// 1. not available, in the same layer
+	// 2. not available, in other layers
+	// 3. available, in local filesystem
 	pp := n.GetRealPath()
-	return filepath.Join(p, pp)
+	if stack, yes := n.ReceivedFile.IsReferencingRequestedImage(); yes {
+		return filepath.Join(n.instance.manager.GetPathByStack(stack), pp), nil
+	}
+	if serial, yes := n.ReceivedFile.IsReferencingLocalFilesystem(); yes {
+		return filepath.Join(n.instance.manager.GetPathBySerial(serial), pp), nil
+	}
+
+	return "", fmt.Errorf("fsnode: unknown file reference [%s]", n.GetName())
 }
 
 func (n *StarlightFsNode) log(filename string, access, complete time.Time) {
@@ -162,7 +179,14 @@ func (n *StarlightFsNode) Readlink(ctx context.Context) ([]byte, syscall.Errno)
 var _ = (fs.NodeOpener)((*StarlightFsNode)(nil))
 
 func (n *StarlightFsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) {
-	r := n.getRealPath()
+	r, err := n.getRealPath()
+	if err != nil {
+		log.G(ctx).WithFields(logrus.Fields{
+			"_s": n.instance.stack,
+			"_r": r,
+		}).Error("open")
+		return nil, 0, syscall.ENODATA
+	}
 
 	access := time.Now()
 	if !n.IsReady() {
@@ -188,7 +212,29 @@ func (n *StarlightFsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle
 var _ = (fs.NodeFsyncer)((*StarlightFsNode)(nil))
 
 func (n *StarlightFsNode) Fsync(ctx context.Context, f fs.FileHandle, flags uint32) syscall.Errno {
-	r := n.getRealPath()
+	r, err := n.getRealPath()
+	if err != nil {
+		log.G(ctx).WithFields(logrus.Fields{
+			"_s": n.instance.stack,
+			"_r": r,
+		}).Error("fsync")
+		return syscall.ENODATA
+	}
+
+	access := time.Now()
+	if !n.IsReady() {
+		n.WaitForReady()
+	}
+	complete := time.Now()
+	name := n.GetName()
+	n.log(name, access, complete)
+
+	log.G(ctx).WithFields(logrus.Fields{
+		"f":  name,
+		"_s": n.instance.stack,
+		"_r": r,
+	}).Trace("fsync")
+
 	fd, err := syscall.Open(r, int(flags), 0)
 	if err != nil {
 		return fs.ToErrno(err)

client/fs/instance.go

@@ -13,7 +13,8 @@ import (
 )
 
 type ImageManager interface {
-	GetPathByLayer(stack int64) string
+	GetPathByStack(stack int64) string
+	GetPathBySerial(stack int64) string
 	LookUpFile(stack int64, filename string) ReceivedFile
 	LogTrace(stack int64, filename string, access, complete time.Time)
 }

client/manager.go

@@ -41,6 +41,8 @@ type Manager struct {
 	cfg *Configuration
 
 	//layers is a map from filesystem serial to receive.ImageLayer object
+	//  - should it require files from layers that are not in this image,
+	//    checkout layers using the serial number
 	layers map[int64]*receive.ImageLayer
 
 	//stackSerialMap is a map convert stack to filesystem serial (on the proxy side),
@@ -75,12 +77,12 @@ func (m *Manager) ignoreStack(stack int64) bool {
 	return m.completedStack[stack]
 }
 
-func (m *Manager) getPathByStack(stack int64) string {
-	return m.layers[m.stackSerialMap[stack]].Local
+func (m *Manager) GetPathByStack(stack int64) string {
+	return m.GetPathBySerial(m.stackSerialMap[stack])
 }
 
-func (m *Manager) GetPathByLayer(stack int64) string {
-	return m.getPathByStack(stack)
+func (m *Manager) GetPathBySerial(serial int64) string {
+	return m.layers[serial].Local
 }
 
 func (m *Manager) LookUpFile(stack int64, filename string) fs.ReceivedFile {
@@ -124,7 +126,7 @@ func (m *Manager) Extract(r *io.ReadCloser) error {
 		}
 
 		// regular extraction
-		p := m.getPathByStack(c.Stack)
+		p := m.GetPathByStack(c.Stack)
 		if err := os.MkdirAll(filepath.Join(p, c.GetBaseDir()), 0755); err != nil {
 			return errors.Wrapf(err, "failed to create directory %s", filepath.Join(p, c.GetBaseDir()))
 		}