Skip to content

Commit

Permalink
Add Debrebuild strategy type (google#231)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wbxyz
wbxyz authored Feb 13, 2025
1 parent 0633e6a commit 9646286
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 5 deletions.
56 changes: 54 additions & 2 deletions pkg/rebuild/debian/strategy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package debian

import (
"path"
"regexp"

"github.com/google/oss-rebuild/internal/textwrap"
Expand Down Expand Up @@ -53,7 +54,7 @@ func (b *DebianPackage) ToWorkflow() *rebuild.WorkflowStrategy {
},
}},
Build: []flow.Step{{
Uses: "debian/build/package",
Uses: "debian/build/debuild",
With: map[string]string{
"targetPath": "{{.Target.Artifact}}",
},
Expand All @@ -66,6 +67,35 @@ func (b *DebianPackage) GenerateFor(t rebuild.Target, be rebuild.BuildEnv) (rebu
return b.ToWorkflow().GenerateFor(t, be)
}

// Debrebuild uses the upstream's generated buildinfo to perform a rebuild.
type Debrebuild struct {
BuildInfo FileWithChecksum `json:"buildinfo" yaml:"buildinfo,omitempty"`
}

func (b *Debrebuild) ToWorkflow() *rebuild.WorkflowStrategy {
return &rebuild.WorkflowStrategy{
Source: []flow.Step{{
Uses: "debian/fetch/buildinfo",
With: map[string]string{
"buildinfoUrl": b.BuildInfo.URL,
"buildinfoMd5": b.BuildInfo.MD5,
},
}},
Build: []flow.Step{{
Uses: "debian/build/debrebuild",
With: map[string]string{
"targetPath": "{{.Target.Artifact}}",
"buildinfo": path.Base(b.BuildInfo.URL),
},
}},
}
}

// Generate generates the instructions for a Debrebuild
func (b *Debrebuild) GenerateFor(t rebuild.Target, be rebuild.BuildEnv) (rebuild.Instructions, error) {
return b.ToWorkflow().GenerateFor(t, be)
}

func init() {
for _, t := range toolkit {
flow.Tools.MustRegister(t)
Expand All @@ -76,6 +106,7 @@ var toolkit = []*flow.Tool{
{
Name: "debian/fetch/sources",
Steps: []flow.Step{{
// TODO: use the FileWithChecksum.MD5 values to verify the downloaded archives.
Runs: textwrap.Dedent(`
wget {{.With.dscUrl}}
{{- if ne .With.nativeUrl "" }}
Expand All @@ -88,6 +119,15 @@ var toolkit = []*flow.Tool{
Needs: []string{"wget", "git"},
}},
},
{
Name: "debian/fetch/buildinfo",
Steps: []flow.Step{{
// TODO: use the FileWithChecksum.MD5 values to verify the downloaded archives.
Runs: textwrap.Dedent(`
wget {{.With.buildinfoUrl}}`)[1:],
Needs: []string{"wget"},
}},
},
{
Name: "debian/deps/install",
Steps: []flow.Step{{
Expand All @@ -98,7 +138,7 @@ var toolkit = []*flow.Tool{
}},
},
{
Name: "debian/build/package",
Name: "debian/build/debuild",
Steps: []flow.Step{{
Runs: textwrap.Dedent(`
cd */
Expand All @@ -110,4 +150,16 @@ var toolkit = []*flow.Tool{
Needs: []string{"build-essential", "fakeroot", "devscripts"},
}},
},
{
Name: "debian/build/debrebuild",
Steps: []flow.Step{{
Runs: textwrap.Dedent(`
debrebuild --buildresult=./out --builder=mmdebstrap {{ .With.buildinfo }}
{{- $expected := regexReplace .With.targetPath "\\+b[0-9]+(_[^_]+\\.deb)$" "$1"}}
{{- if ne $expected .With.targetPath }}
mv /src/{{$expected}} /src/{{.With.targetPath}}
{{- end}}`)[1:],
Needs: []string{"devscripts", "apt-utils", "mmdebstrap"},
}},
},
}
66 changes: 66 additions & 0 deletions pkg/rebuild/debian/strategy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,72 @@ mv /src/pkg_1.0-1_amd64.deb /src/pkg_1.0-1+b1_amd64.deb`,
})
}
}
func TestDebrebuild(t *testing.T) {
tests := []struct {
name string
strategy Debrebuild
target rebuild.Target
env rebuild.BuildEnv
want rebuild.Instructions
}{
{
name: "normal",
strategy: Debrebuild{
BuildInfo: FileWithChecksum{
URL: "https://buildinfos.debian.net/buildinfo-pool/a/acl/acl_2.3.2-2_amd64.buildinfo",
MD5: "deadbeef",
},
},
target: rebuild.Target{
Ecosystem: rebuild.Debian,
Package: "main/acl",
Version: "2.3.1-3",
Artifact: "acl_2.3.1-3_amd64.deb",
},
env: rebuild.BuildEnv{},
want: rebuild.Instructions{
Source: `wget https://buildinfos.debian.net/buildinfo-pool/a/acl/acl_2.3.2-2_amd64.buildinfo`,
Build: `debrebuild --buildresult=./out --builder=mmdebstrap acl_2.3.2-2_amd64.buildinfo`,
OutputPath: "acl_2.3.1-3_amd64.deb",
SystemDeps: []string{"wget", "devscripts", "apt-utils", "mmdebstrap"},
},
},
{
name: "binaryRelease",
strategy: Debrebuild{
BuildInfo: FileWithChecksum{
URL: "https://buildinfos.debian.net/buildinfo-pool/a/acl/acl_2.3.2-2+b1_amd64.buildinfo",
MD5: "deadbeef",
},
},
target: rebuild.Target{
Ecosystem: rebuild.Debian,
Package: "main/acl",
Version: "2.3.1-3+b1",
Artifact: "acl_2.3.1-3+b1_amd64.deb",
},
env: rebuild.BuildEnv{},
want: rebuild.Instructions{
Source: `wget https://buildinfos.debian.net/buildinfo-pool/a/acl/acl_2.3.2-2+b1_amd64.buildinfo`,
Build: `debrebuild --buildresult=./out --builder=mmdebstrap acl_2.3.2-2+b1_amd64.buildinfo
mv /src/acl_2.3.1-3_amd64.deb /src/acl_2.3.1-3+b1_amd64.deb`,
OutputPath: "acl_2.3.1-3+b1_amd64.deb",
SystemDeps: []string{"wget", "devscripts", "apt-utils", "mmdebstrap"},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := tc.strategy.GenerateFor(tc.target, tc.env)
if err != nil {
t.Fatalf("Debrebuild.GenerateFor() failed unexpectedly: %v", err)
}
if diff := cmp.Diff(got, tc.want); diff != "" {
t.Errorf("Debrebuild.GenerateFor() returned diff (-got +want):\n%s", diff)
}
})
}
}

func TestBinaryVersionRegex(t *testing.T) {
tests := []struct {
Expand Down
7 changes: 7 additions & 0 deletions pkg/rebuild/schema/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type StrategyOneOf struct {
CratesIOCargoPackage *cratesio.CratesIOCargoPackage `json:"cratesio_cargo_package,omitempty" yaml:"cratesio_cargo_package,omitempty"`
MavenBuild *maven.MavenBuild `json:"maven_build,omitempty" yaml:"maven_build,omitempty"`
DebianPackage *debian.DebianPackage `json:"debian_package,omitempty" yaml:"debian_package,omitempty"`
Debrebuild *debian.Debrebuild `json:"debrebuild,omitempty" yaml:"debrebuild,omitempty"`
ManualStrategy *rebuild.ManualStrategy `json:"manual,omitempty" yaml:"manual,omitempty"`
WorkflowStrategy *rebuild.WorkflowStrategy `json:"flow,omitempty" yaml:"flow,omitempty"`
}
Expand All @@ -51,6 +52,8 @@ func NewStrategyOneOf(s rebuild.Strategy) StrategyOneOf {
oneof.CratesIOCargoPackage = t
case *debian.DebianPackage:
oneof.DebianPackage = t
case *debian.Debrebuild:
oneof.Debrebuild = t
case *rebuild.ManualStrategy:
oneof.ManualStrategy = t
case *rebuild.WorkflowStrategy:
Expand Down Expand Up @@ -88,6 +91,10 @@ func (oneof *StrategyOneOf) Strategy() (rebuild.Strategy, error) {
num++
s = oneof.DebianPackage
}
if oneof.Debrebuild != nil {
num++
s = oneof.Debrebuild
}
if oneof.ManualStrategy != nil {
num++
s = oneof.ManualStrategy
Expand Down
17 changes: 14 additions & 3 deletions pkg/registry/debian/debian.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
"io"
"net/http"
"path/filepath"
"regexp"
"strings"

Expand All @@ -17,7 +18,8 @@ import (
)

var (
registryURL = "https://deb.debian.org/debian"
registryURL = "https://deb.debian.org/debian/pool/"
buildinfoURL = "https://buildinfos.debian.net/buildinfo-pool/"
binaryReleaseRegexp = regexp.MustCompile(`(\+b[\d\.]+)$`)
)

Expand Down Expand Up @@ -52,14 +54,23 @@ func (r HTTPRegistry) get(ctx context.Context, url string) (io.ReadCloser, error
return resp.Body, nil
}

func PoolURL(component, name, artifact string) string {
func poolSuffix(component, name, artifact string) string {
// Most packages are in a prefix dir matching their first letter.
prefixDir := name[0:1]
// "lib" is such a common prefix that these packages are subdivided into lib* directories.
if strings.HasPrefix(name, "lib") {
prefixDir = name[0:4]
}
return registryURL + fmt.Sprintf("/pool/%s/%s/%s/%s", component, prefixDir, name, artifact)
return fmt.Sprintf("%s/%s/%s/%s", component, prefixDir, name, artifact)
}

func PoolURL(component, name, artifact string) string {
return registryURL + poolSuffix(component, name, artifact)
}

func BuildInfoURL(component, name, artifact string) string {
file := fmt.Sprintf("%s.buildinfo", strings.TrimSuffix(artifact, filepath.Ext(artifact)))
return buildinfoURL + poolSuffix(component, name, file)
}

func guessDSCURL(component, name, version string) string {
Expand Down

0 comments on commit 9646286

Please sign in to comment.