From 6ec4392abccc9e0e85235c3f6f498f4eb8a0f089 Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Thu, 4 Apr 2024 13:12:55 -0400 Subject: [PATCH] Initial resolver working Signed-off-by: Gavin Zhao --- builder/build.go | 16 ++++--- builder/layer.go | 40 ++++++++++++----- builder/manager.go | 101 ++++++++++++++++++++++++++++++++++++++++++- builder/pkg.go | 4 ++ builder/resolver.go | 103 ++++++++++++++++++++++++++++++++++++++++++++ builder/util.go | 13 +++++- go.mod | 2 + go.sum | 6 +++ 8 files changed, 266 insertions(+), 19 deletions(-) create mode 100644 builder/resolver.go diff --git a/builder/build.go b/builder/build.go index e0584e9..856fd9e 100644 --- a/builder/build.go +++ b/builder/build.go @@ -229,12 +229,9 @@ func (p *Package) CopyAssets(h *PackageHistory, o *Overlay) error { return h.WriteXML(histPath) } -func (p *Package) calcDeps(profile *Profile) (deps []string) { +func (p *Package) calcDeps(resolver *Resolver) ([]Dep, error) { // hash = LayersFakeHash - deps = append(deps, "rust") - deps = append(deps, "cargo") - deps = append(deps, "llvm") - return + return resolver.Query(p.Deps, true, true) } // PrepYpkg will do the initial leg work of preparing us for a ypkg build. @@ -491,7 +488,7 @@ func (p *Package) CollectAssets(overlay *Overlay, usr *UserInfo, manifestTarget } // Build will attempt to build the package in the overlayfs system. -func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, manifestTarget string) error { +func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, resolver *Resolver, manifestTarget string) error { slog.Debug("Building package", "name", p.Name, "version", p.Version, "release", p.Release, "type", p.Type, "profile", overlay.Back.Name) @@ -508,7 +505,11 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro // Set up layers caching, only for YPKG if p.Type == PackageTypeYpkg { - deps := p.calcDeps(profile) + deps, err := p.calcDeps(resolver) + if err != nil { + return fmt.Errorf("Failed to calculate dependencies: %w", err) + } + layer := Layer{ deps: deps, profile: profile, @@ -520,6 +521,7 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro return err } overlay.LayerDir = contentPath + slog.Info("Using layer", "hash", layer.Hash()) } else { return errors.New("Under testing of layers feature, XML build is not enabled yet.") } diff --git a/builder/layer.go b/builder/layer.go index b999906..630d3cd 100644 --- a/builder/layer.go +++ b/builder/layer.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "os" "path/filepath" "strings" @@ -12,16 +13,18 @@ import ( ) type Layer struct { - deps []string + deps []Dep profile *Profile back *BackingImage + created bool + hash string } func (l Layer) MarshalJSON() ([]byte, error) { var imageHash string var err error if PathExists(l.back.ImagePath) { - if imageHash, err = hashFile(l.back.ImagePath); err != nil { + if imageHash, err = xxh3128HashFile(l.back.ImagePath); err != nil { return nil, err } // } else if PathExists(l.back.ImagePath) { @@ -33,19 +36,23 @@ func (l Layer) MarshalJSON() ([]byte, error) { } return json.Marshal(struct { - Deps []string + Deps []Dep `json:"deps"` ImageHash string }{Deps: l.deps, ImageHash: imageHash}) } func (l *Layer) Hash() string { - jsonBytes, err := json.Marshal(l) - if err != nil { - return LayersFakeHash - } else { - hashBytes := blake3.Sum256(jsonBytes) - return base64.StdEncoding.EncodeToString(hashBytes[:]) + if l.hash == "" { + jsonBytes, err := json.Marshal(l) + if err != nil { + l.hash = LayersFakeHash + } else { + hashBytes := blake3.Sum256(jsonBytes) + l.hash = base64.StdEncoding.EncodeToString(hashBytes[:]) + } } + return l.hash + } func (l *Layer) BasePath() string { @@ -61,7 +68,15 @@ func (l *Layer) RequestOverlay(notif PidNotifier) (contentPath string, err error } } +func (l *Layer) RemoveIfNotCreated() { + slog.Debug("Layer not fully created, removing...", "path", l.BasePath()) + if !l.created { + os.RemoveAll(l.BasePath()) + } +} + func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) { + slog.Info("Creating layer", "hash", l.Hash()) basePath := l.BasePath() contentPath = filepath.Join(basePath, "content") @@ -136,7 +151,12 @@ func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) { } // Install our dependencies - cmd := fmt.Sprintf("eopkg it -y %s", strings.Join(l.deps, " ")) + pkgs := make([]string, len(l.deps)) + for idx, dep := range l.deps { + pkgs[idx] = dep.Name + } + slog.Debug("Installing dependencies", "size", len(pkgs), "pkgs", pkgs) + cmd := fmt.Sprintf("eopkg it -y %s", strings.Join(pkgs, " ")) if DisableColors { cmd += " -n" } diff --git a/builder/manager.go b/builder/manager.go index 3d67a7f..443c8f1 100644 --- a/builder/manager.go +++ b/builder/manager.go @@ -17,19 +17,25 @@ package builder import ( + "encoding/xml" "errors" "fmt" + "io" "log/slog" + "net/http" "os" "os/signal" + "path" "path/filepath" "strings" "sync" "syscall" "time" + "github.com/getsolus/libeopkg/index" "github.com/getsolus/libosdev/disk" "github.com/go-git/go-git/v5" + "github.com/ulikunitz/xz" "github.com/getsolus/solbuild/cli/log" ) @@ -72,6 +78,7 @@ type Manager struct { pkgManager *EopkgManager // Package manager, if any lock *sync.Mutex // Lock on all operations to prevent.. damage. profile *Profile // The profile we've been requested to use + resolver *Resolver lockfile *LockFile // We track the global lock for each operation didStart bool // Whether we got anything done. @@ -374,7 +381,10 @@ func (m *Manager) Build() error { return err } - return m.pkg.Build(m, m.history, m.GetProfile(), m.pkgManager, m.overlay, m.manifestTarget) + m.InitResolver() + slog.Debug("Successfully initialized resolver") + + return m.pkg.Build(m, m.history, m.GetProfile(), m.pkgManager, m.overlay, m.resolver, m.manifestTarget) } // Chroot will enter the build environment to allow users to introspect it. @@ -478,3 +488,92 @@ func (m *Manager) SetTmpfs(enable bool, size string) { m.Config.TmpfsSize = strings.TrimSpace(size) } } + +func (m *Manager) InitResolver() error { + m.resolver = NewResolver() + + if m.profile == nil { + return errors.New("Profile not initialized!") + } + + profile := m.profile + /// nameToUrl := make(map[string]string) + repos := []string{} + + if strings.Contains(profile.Image, "unstable") { + // nameToUrl["Solus"] = "https://cdn.getsol.us/repo/unstable/eopkg-index.xml.xz" + repos = append(repos, "https://cdn.getsol.us/repo/unstable/eopkg-index.xml.xz") + // repos = append(repos, "https://packages.getsol.us/unstable/eopkg-index.xml.xz") + } else if strings.Contains(profile.Image, "stable") { + // nameToUrl["Solus"] = "https://cdn.getsol.us/repo/shannon/eopkg-index.xml.xz" + repos = append(repos, "https://cdn.getsol.us/repo/shannon/eopkg-index.xml.xz") + } else { + slog.Warn("Unrecognized image name, not adding default repo", "image", profile.Image) + } + + // Realistically, remove can only be * or Solus + // for _, remove := range profile.RemoveRepos { + // if remove == "*" { + // repos = []string{} + // continue + // } + + // if idx := slices.Index(repos, remove); idx != -1 { + // repos = slices.Delete(repos, idx, idx+1) + // } else { + // slog.Warn("Cannot remove noexistent repo", "name", remove) + // } + // } + if len(profile.RemoveRepos) != 0 { + repos = []string{} + if len(profile.RemoveRepos) > 1 { + slog.Warn("Unexpectedly requested removing of more than 1 repo", "removes", profile.RemoveRepos) + } + } + + for _, add := range profile.AddRepos { + if repo := profile.Repos[add]; repo != nil { + repos = append(repos, repo.URI) + } else { + slog.Warn("Cannot add nonexistent repo", "name", add) + } + } + + for _, repo := range repos { + slog.Debug("Fetching repo", "url", repo) + + var r io.Reader + ext := path.Ext(repo) + resp, err := http.Get(repo) + if err != nil { + // slog.Error("Failed to fetch", "url", repo, "error", err) + return fmt.Errorf("Failed to fetch %s: %w", repo, err) + } + slog.Debug("Fetched") + + if ext == ".xz" { + // slog.Debug("Decoding .xz") + if r, err = xz.NewReader(resp.Body); err != nil { + // slog.Error("Failed to init xz reader", "error", err) + return fmt.Errorf("Failed to init xz reader for %s: %w", repo, err) + } + } else if ext == ".xml" { + r = resp.Body + } else { + // slog.Error("Unrecognized repo url extension", "url", repo, "ext", ext) + return fmt.Errorf("Unrecognized repo url extension %s for %s", ext, repo) + } + + dec := xml.NewDecoder(r) + var i index.Index + if err := dec.Decode(&i); err != nil { + // slog.Error("Failed to decode index", "error", err) + return fmt.Errorf("Failed to decode index for %s: %w", repo, err) + } + + m.resolver.AddIndex(&i) + slog.Info("Parsed and added repo to resolver", "url", repo) + } + + return nil +} diff --git a/builder/pkg.go b/builder/pkg.go index 6232eb0..524d9a5 100644 --- a/builder/pkg.go +++ b/builder/pkg.go @@ -62,6 +62,7 @@ type Package struct { Path string // Path to the build spec Sources []source.Source // Each package has 0 or more sources that we fetch CanNetwork bool // Only applicable to ypkg builds + Deps []string } // YmlPackage is a parsed ypkg build file. @@ -71,6 +72,8 @@ type YmlPackage struct { Release int `yaml:"release"` Networking bool `yaml:"networking"` // If set to false (default) we disable networking in the build Source []map[string]string `yaml:"source"` + BuildDeps []string `yaml:"builddeps"` + CheckDeps []string `yaml:"checkdeps"` } // XMLUpdate represents an update in the package history. @@ -220,6 +223,7 @@ func NewYmlPackageFromBytes(by []byte) (*Package, error) { Release: ypkg.Release, Type: PackageTypeYpkg, CanNetwork: ypkg.Networking, + Deps: append(ypkg.BuildDeps, ypkg.CheckDeps...), } for _, row := range ypkg.Source { diff --git a/builder/resolver.go b/builder/resolver.go new file mode 100644 index 0000000..9dc83c8 --- /dev/null +++ b/builder/resolver.go @@ -0,0 +1,103 @@ +package builder + +import ( + "cmp" + "errors" + "fmt" + "slices" + + "github.com/getsolus/libeopkg/index" +) + +type Resolver struct { + // indices []Index + providers map[string]string + nameToPkg map[string]index.Package +} + +type Dep struct { + Name string `json:"name"` + Hash string `json:"hash"` +} + +func NewResolver() (res *Resolver) { + res = &Resolver{ + providers: make(map[string]string), + nameToPkg: make(map[string]index.Package), + } + return +} + +func (r *Resolver) AddIndex(i *index.Index) { + for _, pkg := range i.Packages { + if _, ok := r.nameToPkg[pkg.Name]; !ok { + r.nameToPkg[pkg.Name] = pkg + } + + if pkg.Provides != nil { + for _, provides := range pkg.Provides.PkgConfig { + provider := fmt.Sprintf("pkgconfig(%s)", provides) + if _, ok := r.providers[provider]; !ok { + r.providers[provider] = pkg.Name + } + } + for _, provides := range pkg.Provides.PkgConfig32 { + provider := fmt.Sprintf("pkgconfig32(%s)", provides) + if _, ok := r.providers[provider]; !ok { + r.providers[provider] = pkg.Name + } + } + } + } +} + +func (r *Resolver) Query(pkgs []string, withBase bool, withDevel bool) (res []Dep, err error) { + visited := make(map[string]bool) + + var dfs func(name string) error + dfs = func(name string) error { + if _, ok := r.providers[name]; ok { + name = r.providers[name] + } + + if visited[name] { + return nil + } + + if _, ok := r.nameToPkg[name]; !ok { + return errors.New("Unable to find provider or package " + name) + } + visited[name] = true + + pkg := r.nameToPkg[name] + res = append(res, Dep{Name: pkg.Name, Hash: pkg.PackageHash}) + for _, dep := range r.nameToPkg[name].RuntimeDependencies { + err = dfs(dep.Name) + if err != nil { + return err + } + } + + return nil + } + + if withBase || withDevel { + for _, pkg := range r.nameToPkg { + if withBase && pkg.PartOf == "system.base" { + dfs(pkg.Name) + } else if withDevel && pkg.PartOf == "system.devel" { + dfs(pkg.Name) + } + } + } + + for _, pkg := range pkgs { + err = dfs(pkg) + if err != nil { + return + } + } + + slices.SortFunc(res, func(a, b Dep) int { return cmp.Compare(a.Name, b.Name) }) + return +} diff --git a/builder/util.go b/builder/util.go index ba7fb69..343c955 100644 --- a/builder/util.go +++ b/builder/util.go @@ -31,6 +31,7 @@ import ( "time" "github.com/getsolus/libosdev/commands" + "github.com/zeebo/blake3" ) // ChrootEnvironment is the env used by ChrootExec calls. @@ -273,7 +274,7 @@ func hashFileBytes(path string) ([]byte, error) { } defer f.Close() - h := sha256.New() + h := blake3.New() if _, err := io.Copy(h, f); err != nil { return nil, err } @@ -287,3 +288,13 @@ func hashFile(path string) (string, error) { } return fmt.Sprintf("%x", bytes), nil } + +func xxh3128HashFile(path string) (string, error) { + cmd := exec.Command("xxh128sum", path) + + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to run xxh128sum %s, reason: %w", path, err) + } + return strings.Split(string(output), " ")[0], nil +} diff --git a/go.mod b/go.mod index af06bea..d12d674 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,11 @@ require ( github.com/cavaliergopher/grab/v3 v3.0.1 github.com/cheggaaa/pb/v3 v3.1.5 github.com/coreos/go-systemd/v22 v22.5.0 + github.com/getsolus/libeopkg v0.1.1-0.20240404173503-db4343f6b8f8 github.com/getsolus/libosdev v0.0.0-20181023041421-9ab0f4b463fd github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 + github.com/ulikunitz/xz v0.5.12 gitlab.com/slxh/go/powerline v0.1.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ed56bfe..9de5ec9 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,10 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/getsolus/libeopkg v0.1.1-0.20230924201845-7f2598d34467 h1:pjgeoJiERX+Pv/AVVzVUhwUw9FN1K8UmeKemESceXI4= +github.com/getsolus/libeopkg v0.1.1-0.20230924201845-7f2598d34467/go.mod h1:icOakA4j3f3NmIgRf8+ODZRA8R202hQ+2ZAGhmKEM+0= +github.com/getsolus/libeopkg v0.1.1-0.20240404173503-db4343f6b8f8 h1:I4mFys5UnxS36qT1qN9/5I9kfHZqHrk5fEPQDVh9BZc= +github.com/getsolus/libeopkg v0.1.1-0.20240404173503-db4343f6b8f8/go.mod h1:3bcRCLrRzgeb1bgshVyS5e3Z1uNX7Gi10eqkiioqXgs= github.com/getsolus/libosdev v0.0.0-20181023041421-9ab0f4b463fd h1:QZoSqUIKIFeqhImxNk1cdY7M4n8JVZxTzuhP+Y0DaK8= github.com/getsolus/libosdev v0.0.0-20181023041421-9ab0f4b463fd/go.mod h1:8P4U+IYO8T6nRPLlC6qv1wMFcc0vK0vMVDCuyiFTTLg= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= @@ -100,6 +104,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=