Add netproxy urls to provenance

We also attempt to convert them to purls, where possible.

stacked-commit: true

internal/api/apiservice/rebuild.go

@@ -23,8 +23,10 @@ import (
 	"github.com/google/oss-rebuild/internal/cache"
 	"github.com/google/oss-rebuild/internal/gcb"
 	"github.com/google/oss-rebuild/internal/httpx"
+	"github.com/google/oss-rebuild/internal/netclassify"
 	"github.com/google/oss-rebuild/internal/verifier"
 	"github.com/google/oss-rebuild/pkg/builddef"
+	"github.com/google/oss-rebuild/pkg/proxy/netlog"
 	cratesrb "github.com/google/oss-rebuild/pkg/rebuild/cratesio"
 	debianrb "github.com/google/oss-rebuild/pkg/rebuild/debian"
 	npmrb "github.com/google/oss-rebuild/pkg/rebuild/npm"
@@ -268,12 +270,35 @@ func buildAndAttest(ctx context.Context, deps *RebuildPackageDeps, mux rebuild.R
 		input.Strategy = entry.Strategy
 		buildDefRepo = entry.BuildDefLoc
 	}
+	var proxyURLs *[]string
+	if useProxy {
+		proxyURLs = &[]string{}
+		rc, err := remoteMetadata.Reader(ctx, rebuild.ProxyNetlogAsset.For(t))
+		if err != nil {
+			return errors.Wrap(err, "fetching netlog")
+		}
+		defer rc.Close()
+		var nl netlog.NetworkActivityLog
+		if err := json.NewDecoder(rc).Decode(&nl); err != nil {
+			return errors.Wrap(err, "decoding netlog")
+		}
+		for _, req := range nl.HTTPRequests {
+			url := req.URL().String()
+			if purl, err := netclassify.ClassifyURL(url); errors.Is(err, netclassify.ErrSkipped) {
+				continue
+			} else if errors.Is(err, netclassify.ErrUnclassified) || err != nil {
+				*proxyURLs = append(*proxyURLs, url)
+			} else {
+				*proxyURLs = append(*proxyURLs, purl)
+			}
+		}
+	}
 	if u, err := url.Parse(deps.ServiceRepo.Repo); err != nil {
 		return errors.Wrap(err, "bad ServiceRepo URL")
 	} else if (u.Scheme == "file" || u.Scheme == "") && !deps.PublishForLocalServiceRepo {
 		return errors.Wrap(err, "disallowed file:// ServiceRepo URL")
 	}
-	eqStmt, buildStmt, err := verifier.CreateAttestations(ctx, input, strategy, id, rb, up, deps.LocalMetadataStore, deps.ServiceRepo, deps.PrebuildRepo, buildDefRepo)
+	eqStmt, buildStmt, err := verifier.CreateAttestations(ctx, input, strategy, id, rb, up, deps.LocalMetadataStore, deps.ServiceRepo, deps.PrebuildRepo, buildDefRepo, proxyURLs)
 	if err != nil {
 		return errors.Wrap(err, "creating attestations")
 	}

internal/verifier/attestation.go

@@ -26,7 +26,7 @@ const (
 )
 
 // CreateAttestations creates the SLSA attestations associated with a rebuild.
-func CreateAttestations(ctx context.Context, input rebuild.Input, finalStrategy rebuild.Strategy, id string, rb, up ArtifactSummary, metadata rebuild.AssetStore, serviceLoc, prebuildLoc, buildDefLoc rebuild.Location) (equivalence, build *in_toto.ProvenanceStatementSLSA1, err error) {
+func CreateAttestations(ctx context.Context, input rebuild.Input, finalStrategy rebuild.Strategy, id string, rb, up ArtifactSummary, metadata rebuild.AssetStore, serviceLoc, prebuildLoc, buildDefLoc rebuild.Location, proxyURLs *[]string) (equivalence, build *in_toto.ProvenanceStatementSLSA1, err error) {
 	t, manualStrategy := input.Target, input.Strategy
 	var dockerfile []byte
 	{
@@ -177,6 +177,13 @@ func CreateAttestations(ctx context.Context, input rebuild.Input, finalStrategy
 			},
 		},
 	}
+	if proxyURLs != nil {
+		proxyURLBytes, err := json.Marshal(*proxyURLs)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "marshalling proxy URLs")
+		}
+		stmt.Predicate.RunDetails.Byproducts = append(stmt.Predicate.RunDetails.Byproducts, slsa1.ResourceDescriptor{Name: "urls.json", Content: proxyURLBytes})
+	}
 	return eqStmt, stmt, nil
 }
 

internal/verifier/attestation_test.go

@@ -69,7 +69,7 @@ func TestCreateAttestations(t *testing.T) {
 		serviceLoc := rebuild.Location{Repo: "https://github.com/google/oss-rebuild", Ref: "v0.0.0-202501010000-feeddeadbeef00"}
 		prebuildLoc := rebuild.Location{Repo: "https://github.com/google/oss-rebuild", Ref: "v0.0.0-202401010000-feeddeadbeef99"}
 		buildDefLoc := rebuild.Location{Repo: "https://github.com/google/oss-rebuild", Ref: "b33eec7134eff8a16cb902b80e434de58bf37e2c", Dir: "definitions/cratesio/bytes/1.0.0/bytes-1.0.0.crate/build.yaml"}
-		eqStmt, buildStmt, err := CreateAttestations(ctx, input, strategy, "test-id", rbSummary, upSummary, metadata, serviceLoc, prebuildLoc, buildDefLoc)
+		eqStmt, buildStmt, err := CreateAttestations(ctx, input, strategy, "test-id", rbSummary, upSummary, metadata, serviceLoc, prebuildLoc, buildDefLoc, &[]string{"pkg:foo/bar@baz"})
 		if err != nil {
 			t.Fatalf("Unexpected error: %v", err)
 		}
@@ -216,6 +216,10 @@ func TestCreateAttestations(t *testing.T) {
         {
           "name": "steps.json",
           "content": "W3sibmFtZSI6Imdjci5pby9mb28vYmFyIiwic2NyaXB0IjoiLi9iYXIifV0="
+        },
+        {
+          "name": "urls.json",
+          "content": "WyJwa2c6Zm9vL2JhckBiYXoiXQ=="
         }
       ]
     }

pkg/proxy/netlog/netlog.go

@@ -6,6 +6,7 @@ package netlog
 import (
 	"net"
 	"net/http"
+	"net/url"
 	"sync"
 
 	"github.com/elazarl/goproxy"
@@ -18,6 +19,10 @@ type HTTPRequestLog struct {
 	Path   string
 }
 
+func (l HTTPRequestLog) URL() *url.URL {
+	return &url.URL{Scheme: l.Scheme, Host: l.Host, Path: l.Path}
+}
+
 type NetworkActivityLog struct {
 	HTTPRequests []HTTPRequestLog
 }