UPDATE: Helm configurations and error handling (#48)

* add namespaces for helm chart

* add better error handling

* add DOCKER_CONFIG env

Dockerfile

@@ -1,32 +1,39 @@
-FROM golang:1.18 AS starlight-proxy-build
+#########################################################################
+# starlight-proxy
+#########################################################################
+FROM golang:1.20 AS starlight-proxy-build
 
 WORKDIR /go/src/app
 COPY . .
 
 ENV GO111MODULE=on
+ENV PRODUCTION=true
 
 RUN make change-version-number set-production starlight-proxy-for-alpine
-
-#CMD ["/go/src/app/out/starlight-proxy"]
+#########################################################################
 FROM alpine:3.12 AS starlight-proxy
 
 COPY --from=starlight-proxy-build /go/src/app/out/ /opt/
 WORKDIR /opt
 EXPOSE 8090
 CMD ["/opt/starlight-proxy"]
 
-FROM golang:1.18 AS starlight-cli-build
+#########################################################################
+# ctr-starlight
+#########################################################################
+FROM golang:1.20 AS starlight-cli-build
 
 WORKDIR /go/src/app
 COPY . .
 
 ENV GO111MODULE=on
+ENV PRODUCTION=true
 
 RUN make change-version-number set-production ctr-starlight-for-alpine
-
+#########################################################################
 FROM alpine:3.12 AS starlight-cli
 
 COPY --from=starlight-cli-build /go/src/app/out/ /opt/
 WORKDIR /opt
 EXPOSE 8090
-CMD ["/opt/ctr-starlight"]
+CMD ["/opt/ctr-starlight"]

Makefile

@@ -9,7 +9,6 @@ VERSIONNUMBER=$(shell echo $(VERSION) | sed 's/v//g')
 COMPILEDATE=$(shell date +%Y%m%d)
 
 
-.PHONY: build clean starlight-proxy starlight-daemon ctr-starlight test
 .SILENT: install-systemd-service
 
 ######################################################################
@@ -57,6 +56,12 @@ starlight-proxy-for-alpine:
 helm-package:
 	helm package ./demo/chart --version $(VERSIONNUMBER) -d /tmp
 
+.PHONE: helm-template
+helm-template:
+	@if [ ! -d ./sandbox ]; then mkdir ./sandbox; fi
+	@if [ ! -f ./sandbox/values.yaml ]; then cp ./demo/values.yaml ./sandbox/values.yaml; fi
+	@helm template ./demo/chart --version $(VERSIONNUMBER) -f ./sandbox/values.yaml
+
 .PHONY: push-helm-package
 push-helm-package:
 	helm push /tmp/starlight-$(VERSIONNUMBER).tgz oci://ghcr.io/mc256/starlight/

client/client.go

@@ -469,10 +469,10 @@ func (c *Client) UploadTraces(proxyCfg string, tc *fs.TraceCollection) error {
 }
 
 type PullFinishedMessage struct {
-	img       *images.Image
-	imageSize int64
-	base      string
-	err       error
+	img  *images.Image
+	meta *common.DeltaImageMetadata
+	base string
+	err  error
 }
 
 // for testing only
@@ -489,7 +489,7 @@ func (c *Client) pullImageGrpc(ns, base, ref, proxy string, ret *chan PullFinish
 	// connect to containerd
 	ctr, err := containerd.New(c.cfg.Containerd, containerd.WithDefaultNamespace(ns))
 	if err != nil {
-		*ret <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to connect to containerd")}
+		*ret <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to connect to containerd")}
 		return
 	}
 	defer ctr.Close()
@@ -498,7 +498,7 @@ func (c *Client) pullImageGrpc(ns, base, ref, proxy string, ret *chan PullFinish
 	var baseImg containerd.Image
 	baseImg, err = c.FindBaseImage(ctr, base, ref)
 	if err != nil {
-		*ret <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to identify base image")}
+		*ret <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to identify base image")}
 		return
 	}
 
@@ -524,19 +524,19 @@ func (c *Client) PullImage(
 	reqFilter := getImageFilter(ref, false)
 	img, err := c.findImage(ctr, reqFilter)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to check requested image %s", ref)}
+		*ready <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to check requested image %s", ref)}
 		return
 	}
 
 	if img != nil {
 		labels := img.Labels()
 		if _, has := labels[util.ContentLabelCompletion]; has {
 			if _, err = c.LoadImage(ctr, img.Target().Digest); err != nil {
-				*ready <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to load image %s", ref)}
+				*ready <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to load image %s", ref)}
 				return
 			}
 			meta := img.Metadata()
-			*ready <- PullFinishedMessage{&meta, 0, "", fmt.Errorf("requested image %s already exists", ref)}
+			*ready <- PullFinishedMessage{&meta, nil, "", fmt.Errorf("requested image %s already exists", ref)}
 			return
 
 		}
@@ -549,7 +549,7 @@ func (c *Client) PullImage(
 			log.G(c.ctx).
 				WithField("image", ref).
 				Info("failed to remove incomplete image")
-			*ready <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to remove unfinished image %s", ref)}
+			*ready <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to remove unfinished image %s", ref)}
 			return
 		}
 	}
@@ -569,7 +569,7 @@ func (c *Client) PullImage(
 	// pull image
 	body, res, err := p.DeltaImage(baseRef, ref, platform)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, "", errors.Wrapf(err, "failed to pull image %s", ref)}
+		*ready <- PullFinishedMessage{nil, nil, "", errors.Wrapf(err, "failed to pull image %s", ref)}
 		return
 	}
 	defer func() {
@@ -628,53 +628,53 @@ func (c *Client) PullImage(
 	// manifest
 	buf, err = c.readBody(body, res.ManifestSize)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to read manifest")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to read manifest")}
 		return
 	}
 	manifest, man, err = c.handleManifest(buf)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to handle manifest")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to handle manifest")}
 		return
 	}
 	err = c.storeManifest(cs, pcn, res.Digest, ref,
 		manifest.Config.Digest.String(), res.StarlightDigest,
 		man)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to store manifest")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to store manifest")}
 		return
 	}
 
 	// config
 	buf, err = c.readBody(body, res.ConfigSize)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to read config")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to read config")}
 		return
 	}
 	imageConfig, con, err = c.handleConfig(buf)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to handle config")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to handle config")}
 		return
 	}
 	err = c.storeConfig(cs, pcn, ref, manifest.Config.Digest, con)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to store config")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to store config")}
 		return
 	}
 
 	// starlight header
 	buf, err = c.readBody(body, res.StarlightHeaderSize)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to read starlight header")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to read starlight header")}
 		return
 	}
 	star, sta, err := c.handleStarlightHeader(buf)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to handle starlight header")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to handle starlight header")}
 		return
 	}
 	err = c.storeStarlightHeader(cs, pcn, ref, res.StarlightDigest, sta)
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to store starlight header")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to store starlight header")}
 		return
 	}
 
@@ -695,7 +695,7 @@ func (c *Client) PullImage(
 		UpdatedAt: time.Now(),
 	})
 	if err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to create image %s", ref)}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to create image %s", ref)}
 		return
 	}
 
@@ -721,13 +721,13 @@ func (c *Client) PullImage(
 			log.G(c.ctx).
 				WithError(err).
 				Error("failed to set optimizer on")
-			*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to enable optimizer")}
+			*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to enable optimizer")}
 			return
 		}
 	}
 
 	if err = star.PrepareDirectories(c); err != nil {
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to initialize directories")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to initialize directories")}
 		return
 	}
 
@@ -739,7 +739,7 @@ func (c *Client) PullImage(
 		log.G(c.ctx).
 			WithError(err).
 			Error("failed to create snapshots")
-		*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to create snapshots")}
+		*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to create snapshots")}
 		return
 	}
 
@@ -751,7 +751,7 @@ func (c *Client) PullImage(
 	// close(*ready)
 	//
 	// wola! we are done here.
-	*ready <- PullFinishedMessage{&ctrImg, res.ContentLength, baseRef, nil}
+	*ready <- PullFinishedMessage{&ctrImg, res, baseRef, nil}
 
 	// 6. Extract file content
 	// download content
@@ -761,7 +761,7 @@ func (c *Client) PullImage(
 
 	if err = star.Extract(&body); err != nil {
 		if ready != nil { // second signal
-			*ready <- PullFinishedMessage{nil, 0, baseRef, errors.Wrapf(err, "failed to extract starlight image")}
+			*ready <- PullFinishedMessage{nil, nil, baseRef, errors.Wrapf(err, "failed to extract starlight image")}
 		}
 		return
 	}
@@ -792,7 +792,7 @@ func (c *Client) PullImage(
 	}
 
 	if ready != nil { // second signal
-		*ready <- PullFinishedMessage{&ctrImg, res.ContentLength, baseRef, nil}
+		*ready <- PullFinishedMessage{&ctrImg, res, baseRef, nil}
 	}
 }
 

client/grpc.go

@@ -96,32 +96,40 @@ func (s *StarlightDaemonAPIServer) PullImage(ctx context.Context, ref *pb.ImageR
 
 	if ret.err != nil {
 		if ret.img != nil {
+			// requested image is already pulled
 			return &pb.ImagePullResponse{
 				Success:        true,
 				Message:        ret.err.Error(),
 				BaseImage:      ret.base,
-				TotalImageSize: ret.imageSize,
+				TotalImageSize: ret.meta.ContentLength,
 			}, nil
 		} else {
 			return &pb.ImagePullResponse{
 				Success:        false,
 				Message:        ret.err.Error(),
 				BaseImage:      ret.base,
-				TotalImageSize: ret.imageSize,
+				TotalImageSize: ret.meta.ContentLength,
 			}, nil
 		}
 	}
 
 	// wait for the entire delta image to be pulled to the local filesystem
 	// holding on the second signal from the pullImageGrpc goroutine
-	if ref.DisableEarlyStart { 
+	if ref.DisableEarlyStart {
 		ret := <-ready // second signal
-		if ret.err != nil {
+		if ret.img != nil {
+			return &pb.ImagePullResponse{
+				Success:        true,
+				Message:        ret.err.Error(),
+				BaseImage:      ret.base,
+				TotalImageSize: ret.meta.ContentLength,
+			}, nil
+		} else {
 			return &pb.ImagePullResponse{
 				Success:        false,
 				Message:        ret.err.Error(),
 				BaseImage:      ret.base,
-				TotalImageSize: ret.imageSize,
+				TotalImageSize: ret.meta.ContentLength,
 			}, nil
 		}
 	}

cmd/ctr-starlight/listproxy/list_proxy.go

@@ -59,8 +59,7 @@ func Action(context *cli.Context) (err error) {
 	}
 
 	// send to proxy server
-	err = listProxyProfile(pb.NewDaemonClient(conn))
-	if err != nil {
+	if err = listProxyProfile(pb.NewDaemonClient(conn)); err != nil {
 		return err
 	}
 

cmd/ctr-starlight/main.go

@@ -45,9 +45,10 @@ func init() {
 func main() {
 	app := NewApp()
 	if err := app.Run(os.Args); err != nil {
-		_, _ = fmt.Fprintf(os.Stderr, "ctr-starlight: \n%v\n", err)
+		_, _ = fmt.Fprintf(os.Stderr, "ctr-starlight: %v\n", err)
 		os.Exit(1)
 	}
+	os.Exit(0)
 }
 
 func NewApp() *cli.App {
@@ -93,7 +94,7 @@ https://github.com/mc256/starlight
 			Required:    false,
 		},
 	}
-	app.Commands = append([]*cli.Command{
+	app.Commands = []*cli.Command{
 		cmdVersion.Command(),   // 1. confirm the version of starlight-daemon
 		cmdAddProxy.Command(),  // 2. add starlight proxy to the daemon
 		cmdListProxy.Command(), // 3. list proxy profiles in daemon
@@ -103,7 +104,7 @@ https://github.com/mc256/starlight
 		cmdOptimizer.Command(), // 7. turn on/off filesystem traces
 		cmdReport.Command(),    // 8. upload filesystem traces to starlight proxy
 		cmdPull.Command(),      // 9. pull starlight image
-	})
+	}
 
 	return app
 }

cmd/ctr-starlight/notify/notify.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+
 	"github.com/containerd/containerd/log"
 	"github.com/google/go-containerregistry/pkg/name"
 	pb "github.com/mc256/starlight/client/api"
@@ -18,11 +19,10 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 )
 
-func notify(client pb.DaemonClient, req *pb.NotifyRequest, quiet bool) {
+func notify(client pb.DaemonClient, req *pb.NotifyRequest, quiet bool) error {
 	resp, err := client.NotifyProxy(context.Background(), req)
 	if err != nil {
-		fmt.Printf("notify starlight proxy server failed: %v\n", err)
-		return
+		return fmt.Errorf("notify starlight proxy server failed: %v", err)
 	}
 	if resp.Success {
 		if !quiet {
@@ -31,6 +31,7 @@ func notify(client pb.DaemonClient, req *pb.NotifyRequest, quiet bool) {
 	} else {
 		fmt.Printf("notify starlight proxy server failed: %v\n", resp)
 	}
+	return nil
 }
 
 func SharedAction(ctx context.Context, c *cli.Context, reference name.Reference) (err error) {
@@ -45,13 +46,11 @@ func SharedAction(ctx context.Context, c *cli.Context, reference name.Reference)
 	defer conn.Close()
 
 	// notify
-	notify(pb.NewDaemonClient(conn), &pb.NotifyRequest{
+	return notify(pb.NewDaemonClient(conn), &pb.NotifyRequest{
 		ProxyConfig: c.String("profile"),
 		Insecure:    c.Bool("insecure") || c.Bool("insecure-destination"),
 		Reference:   reference.String(),
 	}, c.Bool("quiet"))
-
-	return nil
 }
 
 func Action(ctx context.Context, c *cli.Context) (err error) {