Overlays/Mounting working, need to hook up dep calculation

Signed-off-by: Gavin Zhao <[email protected]>

Makefile

@@ -53,4 +53,7 @@ release:
 
 .PHONY: clean
 clean:
-	rm -rf $(CURDIR)/bin
+	rm -rvf $(CURDIR)/bin
+	rm -rvf $(CURDIR)/*.eopkg
+	rm -rvf $(CURDIR)/pspec_x86_64.xml
+	rm -rvf $(CURDIR)/abi*

builder/build.go

@@ -229,8 +229,11 @@ func (p *Package) CopyAssets(h *PackageHistory, o *Overlay) error {
 	return h.WriteXML(histPath)
 }
 
-func (p *Package) calcHashAndDeps() (hash string, deps []string, profile *Profile) {
-	hash = LayersFakeHash
+func (p *Package) calcDeps(profile *Profile) (deps []string) {
+	// hash = LayersFakeHash
+	deps = append(deps, "rust")
+	deps = append(deps, "cargo")
+	deps = append(deps, "llvm")
 	return
 }
 
@@ -251,32 +254,32 @@ func (p *Package) PrepYpkg(notif PidNotifier, usr *UserInfo, pman *EopkgManager,
 		return fmt.Errorf("Failed to write packager file %s, reason: %w\n", fp, err)
 	}
 
-	wdir := p.GetWorkDirInternal()
-	ymlFile := filepath.Join(wdir, filepath.Base(p.Path))
+	// wdir := p.GetWorkDirInternal()
+	// ymlFile := filepath.Join(wdir, filepath.Base(p.Path))
 
-	cmd := fmt.Sprintf("ypkg-install-deps -f %s", ymlFile)
-	if DisableColors {
-		cmd += " -n"
-	}
+	// cmd := fmt.Sprintf("ypkg-install-deps -f %s", ymlFile)
+	// if DisableColors {
+	// 	cmd += " -n"
+	// }
 
-	// Install build dependencies
-	slog.Debug("Installing build dependencies", "file", ymlFile)
+	// // Install build dependencies
+	// slog.Debug("Installing build dependencies", "file", ymlFile)
 
-	if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
-		return fmt.Errorf("Failed to install build dependencies %s, reason: %w\n", ymlFile, err)
-	}
+	// if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
+	// 	return fmt.Errorf("Failed to install build dependencies %s, reason: %w\n", ymlFile, err)
+	// }
 
-	notif.SetActivePID(0)
+	// notif.SetActivePID(0)
 
-	// Cleanup now
-	slog.Debug("Stopping D-BUS")
+	// // Cleanup now
+	// slog.Debug("Stopping D-BUS")
 
-	if err := pman.StopDBUS(); err != nil {
-		return fmt.Errorf("Failed to stop d-bus, reason: %w\n", err)
-	}
+	// if err := pman.StopDBUS(); err != nil {
+	// 	return fmt.Errorf("Failed to stop d-bus, reason: %w\n", err)
+	// }
 
 	// Chwn the directory before bringing up sources
-	cmd = fmt.Sprintf("chown -R %s:%s %s", BuildUser, BuildUser, BuildUserHome)
+	cmd := fmt.Sprintf("chown -R %s:%s %s", BuildUser, BuildUser, BuildUserHome)
 	if err := ChrootExec(notif, overlay.MountPoint, cmd); err != nil {
 		return fmt.Errorf("Failed to set home directory permissions, reason: %w\n", err)
 	}
@@ -503,6 +506,24 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 
 	ChrootEnvironment = env
 
+	// Set up layers caching, only for YPKG
+	if p.Type == PackageTypeYpkg {
+		deps := p.calcDeps(profile)
+		layer := Layer{
+			deps:    deps,
+			profile: profile,
+			back:    overlay.Back,
+		}
+
+		contentPath, err := layer.RequestOverlay(notif)
+		if err != nil {
+			return err
+		}
+		overlay.LayerDir = contentPath
+	} else {
+		return errors.New("Under testing of layers feature, XML build is not enabled yet.")
+	}
+
 	// Set up environment
 	if err := overlay.CleanExisting(); err != nil {
 		return err
@@ -531,34 +552,34 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 		return err
 	}
 
-	// Set up package manager
-	if err := pman.Init(); err != nil {
-		return err
-	}
+	// // Set up package manager
+	// if err := pman.Init(); err != nil {
+	// 	return err
+	// }
 
-	// Bring up dbus to do Things
-	slog.Debug("Starting D-BUS")
+	// // Bring up dbus to do Things
+	// slog.Debug("Starting D-BUS")
 
-	if err := pman.StartDBUS(); err != nil {
-		return fmt.Errorf("Failed to start d-bus, reason: %w\n", err)
-	}
+	// if err := pman.StartDBUS(); err != nil {
+	// 	return fmt.Errorf("Failed to start d-bus, reason: %w\n", err)
+	// }
 
-	// Get the repos in place before asserting anything
-	if err := pman.ConfigureRepos(notif, overlay, profile); err != nil {
-		return fmt.Errorf("Configuring repositories failed, reason: %w\n", err)
-	}
+	// // Get the repos in place before asserting anything
+	// if err := pman.ConfigureRepos(notif, overlay, profile); err != nil {
+	// 	return fmt.Errorf("Configuring repositories failed, reason: %w\n", err)
+	// }
 
-	slog.Debug("Upgrading system base")
+	// slog.Debug("Upgrading system base")
 
-	if err := pman.Upgrade(); err != nil {
-		return fmt.Errorf("Failed to upgrade rootfs, reason: %w\n", err)
-	}
+	// if err := pman.Upgrade(); err != nil {
+	// 	return fmt.Errorf("Failed to upgrade rootfs, reason: %w\n", err)
+	// }
 
-	slog.Debug("Asserting system.devel component installation")
+	// slog.Debug("Asserting system.devel component installation")
 
-	if err := pman.InstallComponent("system.devel"); err != nil {
-		return fmt.Errorf("Failed to assert system.devel, reason: %w\n", err)
-	}
+	// if err := pman.InstallComponent("system.devel"); err != nil {
+	// 	return fmt.Errorf("Failed to assert system.devel, reason: %w\n", err)
+	// }
 
 	// Ensure all directories are in place
 	if err := p.CreateDirs(overlay); err != nil {

builder/layer.go

@@ -1,34 +1,69 @@
 package builder
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"log/slog"
 	"path/filepath"
 	"strings"
+
+	"github.com/zeebo/blake3"
 )
 
 type Layer struct {
-	hash    string
 	deps    []string
 	profile *Profile
 	back    *BackingImage
 }
 
+func (l Layer) MarshalJSON() ([]byte, error) {
+	var imageHash string
+	var err error
+	if PathExists(l.back.ImagePath) {
+		if imageHash, err = hashFile(l.back.ImagePath); err != nil {
+			return nil, err
+		}
+		// } else if PathExists(l.back.ImagePath) {
+		// 	if imageHash, err = hashFile(l.back.ImagePath); err != nil {
+		// 		return
+		// 	}
+	} else {
+		return nil, fmt.Errorf("Backing image doens't exist at %s", l.back.ImagePath)
+	}
+
+	return json.Marshal(struct {
+		Deps      []string
+		ImageHash string
+	}{Deps: l.deps, ImageHash: imageHash})
+}
+
+func (l *Layer) Hash() string {
+	jsonBytes, err := json.Marshal(l)
+	if err != nil {
+		return LayersFakeHash
+	} else {
+		hashBytes := blake3.Sum256(jsonBytes)
+		return base64.StdEncoding.EncodeToString(hashBytes[:])
+	}
+}
+
 func (l *Layer) BasePath() string {
-	return filepath.Join(LayersDir, l.hash)
+	return filepath.Join(LayersDir, l.Hash())
 }
 
-func (l *Layer) RequestOverlay(notif PidNotifier, pman *EopkgManager) (err error, ovly *Overlay) {
-	if PathExists(filepath.Join(l.BasePath(), "content")) {
-		return l.Create(notif, pman)
+func (l *Layer) RequestOverlay(notif PidNotifier) (contentPath string, err error) {
+	contentPath = filepath.Join(l.BasePath(), "content")
+	if !PathExists(contentPath) || l.Hash() == LayersFakeHash {
+		return l.Create(notif)
 	} else {
 		return
 	}
 }
 
-func (l *Layer) Create(notif PidNotifier, pman *EopkgManager) (err error, ovly *Overlay) {
+func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) {
 	basePath := l.BasePath()
-	contentPath := filepath.Join(basePath, "content")
+	contentPath = filepath.Join(basePath, "content")
 
 	depsOverlay := Overlay{
 		Back:           l.back,
@@ -63,6 +98,8 @@ func (l *Layer) Create(notif PidNotifier, pman *EopkgManager) (err error, ovly *
 	}
 	defer depsOverlay.DeactivateRoot()
 
+	pman := NewEopkgManager(notif, depsOverlay.MountPoint)
+
 	// Init pman
 	if err = pman.Init(); err != nil {
 		err = fmt.Errorf("Failed to init pman, reason: %w", err)

builder/overlay.go

@@ -38,6 +38,7 @@ type Overlay struct {
 	BaseDir    string // BaseDir is the base directory containing the root
 	WorkDir    string // WorkDir is the overlayfs workdir lock
 	UpperDir   string // UpperDir is where real inode changes happen (tmp)
+	LayerDir   string
 	ImgDir     string // Where the profile is mounted (ro)
 	MountPoint string // The actual mount point for the union'd directories
 	LockPath   string // Path to the lockfile for this overlay
@@ -167,12 +168,18 @@ func (o *Overlay) Mount() error {
 	o.mountedImg = true
 
 	// Now mount the overlayfs
-	slog.Debug("Mounting overlayfs", "upper", o.UpperDir, "lower", o.ImgDir,
+	var lowerDir string
+	if len(o.LayerDir) == 0 {
+		lowerDir = o.ImgDir
+	} else {
+		lowerDir = o.LayerDir + ":" + o.ImgDir
+	}
+	slog.Debug("Mounting overlayfs", "upper", o.UpperDir, "lower", lowerDir,
 		"workdir", o.WorkDir, "target", o.MountPoint)
 
 	// Mounting overlayfs..
 	err := mountMan.Mount("overlay", o.MountPoint, "overlay",
-		fmt.Sprintf("lowerdir=%s", o.ImgDir),
+		fmt.Sprintf("lowerdir=%s", lowerDir),
 		fmt.Sprintf("upperdir=%s", o.UpperDir),
 		fmt.Sprintf("workdir=%s", o.WorkDir))
 	// Check non-fatal..

builder/util.go

@@ -20,6 +20,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 	"os/exec"
@@ -264,3 +265,25 @@ func ValidMemSize(s string) bool {
 
 	return false
 }
+
+func hashFileBytes(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil, err
+	}
+	return h.Sum(nil), nil
+}
+
+func hashFile(path string) (string, error) {
+	bytes, err := hashFileBytes(path)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", bytes), nil
+}

go.mod

@@ -16,6 +16,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+
 require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
@@ -39,6 +41,7 @@ require (
 	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/solus-project/libosdev v0.0.0-20171113084438-39032fc50772 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/zeebo/blake3 v0.2.3
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.19.0 // indirect

go.sum

@@ -59,6 +59,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -101,6 +103,12 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 gitlab.com/slxh/go/powerline v0.1.0 h1:/3lwpGRD5yW9HFS/hammtCI4kvtjKw8E1dcpHS9Udx8=
 gitlab.com/slxh/go/powerline v0.1.0/go.mod h1:vBTN83xoDyGejdTeZkMGs8l/qZvOjpUkRMYrthNhqJE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=