Update WAF to 1.15.0

.github/workflows/test.yml

@@ -76,10 +76,11 @@ jobs:
           # Run the tests with gotestsum
           env CGO_ENABLED=${{ matrix.cgo_enabled }} ./gotestsum -- -v -count=10 -shuffle=on ./...
 
-  linux-arm64:
+  linux-other:
     runs-on: ubuntu-latest
     strategy:
       matrix:
+        arch: ["arm64"]
         cgo_enabled: [ "0", "1" ] # test it compiles with and without the cgo
       fail-fast: false
     steps:
@@ -88,13 +89,13 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
-          key: go-pkg-mod-${{ hashFiles('**/go.sum') }}
-          restore-keys: go-pkg-mod-
+          key: go-pkg-mod-${{ matrix.arch }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-pkg-mod-${{ matrix.arch }} go-pkg-mod-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:
-          platforms: arm64
-      - run: docker run --platform=linux/arm64 -v $PWD:$PWD -w $PWD -eCGO_ENABLED=${{ matrix.cgo_enabled }} -eDD_APPSEC_WAF_TIMEOUT=$DD_APPSEC_WAF_TIMEOUT golang go test -v -count=10 -shuffle=on ./...
+          platforms: ${{ matrix.arch }}
+      - run: docker run --platform=linux/${{ matrix.arch }} -v${HOME}/go/pkg/mod:/root/go/pkg/mod -v $PWD:$PWD -w $PWD -eCGO_ENABLED=${{ matrix.cgo_enabled }} -eDD_APPSEC_WAF_TIMEOUT=$DD_APPSEC_WAF_TIMEOUT golang go test -v -count=10 -shuffle=on ./...
 
   # A simple join target to simplify setting up branch protection settings in GH.
   done:
@@ -103,7 +104,7 @@ jobs:
     needs:
       - native
       - golang-linux-container
-      - linux-arm64
+      - linux-other
     steps:
       - name: Done
         run: echo "Done!"

_tools/libddwaf-updater/go.mod

@@ -0,0 +1,15 @@
+module github.com/DataDog/go-libddwaf/libddwaf-updater
+
+go 1.18
+
+require (
+	github.com/bitfield/script v0.22.0
+	github.com/google/go-github/v56 v56.0.0
+)
+
+require (
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/itchyny/gojq v0.12.12 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
+	mvdan.cc/sh/v3 v3.6.0 // indirect
+)

_tools/libddwaf-updater/go.sum

@@ -0,0 +1,42 @@
+github.com/bitfield/script v0.22.0 h1:LA7QHuEsXMPD52YLtxWrlqCCy+9FOpzNYfsRHC5Gsrc=
+github.com/bitfield/script v0.22.0/go.mod h1:ms4w+9B8f2/W0mbsgWDVTtl7K94bYuZc3AunnJC4Ebs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v56 v56.0.0 h1:TysL7dMa/r7wsQi44BjqlwaHvwlFlqkK8CtBWCX3gb4=
+github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
+github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA=
+github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE=
+github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
+github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+mvdan.cc/editorconfig v0.2.0/go.mod h1:lvnnD3BNdBYkhq+B4uBuFFKatfp02eB6HixDvEz91C0=
+mvdan.cc/sh/v3 v3.6.0 h1:gtva4EXJ0dFNvl5bHjcUEvws+KRcDslT8VKheTYkbGU=
+mvdan.cc/sh/v3 v3.6.0/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA=

_tools/libddwaf-updater/update.go

@@ -0,0 +1,280 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package main
+
+import (
+	gotar "archive/tar"
+	"compress/gzip"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/bitfield/script"
+	"github.com/google/go-github/v56/github"
+)
+
+var (
+	rootDir        string
+	libDir         string
+	versionFile    string
+	currentVersion string
+)
+
+func main() {
+	force := os.Args[1] == "--force"
+
+	gh := github.NewClient(nil)
+
+	release, _, err := gh.Repositories.GetLatestRelease(context.Background(), "DataDog", "libddwaf")
+	if err != nil {
+		panic(err)
+	}
+
+	version := *release.TagName
+	if version == currentVersion {
+		fmt.Printf("Already up-to-date with v%s\n", version)
+		if force {
+			fmt.Println("--force is set, re-downloading assets anyway!")
+		} else {
+			return
+		}
+	} else {
+		fmt.Printf("Will upgrade from v%s to v%s\n", currentVersion, version)
+	}
+
+	assets := make(map[string]*github.ReleaseAsset, len(release.Assets))
+	for _, asset := range release.Assets {
+		if asset.Name == nil {
+			continue
+		}
+		assets[*asset.Name] = asset
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(len(targets))
+	for _, tgt := range targets {
+		embedDir := path.Join(libDir, fmt.Sprintf("%s-%s", tgt.os, tgt.arch))
+		created := false
+		if _, err = os.Stat(embedDir); errors.Is(err, os.ErrNotExist) {
+			if err = os.MkdirAll(embedDir, 0755); err != nil {
+				panic(err)
+			}
+			created = true
+		}
+		if created || force {
+			createEmbedSource(tgt)
+		}
+		go handleTarget(&wg, version, tgt, embedDir, assets)
+	}
+
+	wg.Wait()
+
+	file, err := os.OpenFile(versionFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		panic(err)
+	}
+	written := 0
+	for written < len(version) {
+		wrote, err := file.WriteString(version[written:])
+		if err != nil {
+			panic(err)
+		}
+		written += wrote
+	}
+
+	fmt.Println("All done! Don't forget to check in changes to include/ and internal/vendor/, check the libddwaf upgrade guide to update bindings!")
+}
+
+func createEmbedSource(tgt target) {
+	ext := "so"
+	if tgt.os == "darwin" {
+		ext = "dylib"
+	}
+
+	gosource := strings.Join(
+		[]string{
+			"// Unless explicitly stated otherwise all files in this repository are licensed",
+			"// under the Apache License Version 2.0.",
+			"// This product includes software developed at Datadog (https://www.datadoghq.com/).",
+			"// Copyright 2016-present Datadog, Inc.",
+			"",
+			fmt.Sprintf("//go:build %s && %s && !go1.22", tgt.os, tgt.arch),
+			"package vendor",
+			"",
+			`import _ "embed" // Needed for go:embed`,
+			"",
+			fmt.Sprintf("//go:embed %s-%s/libddwaf.%s", tgt.os, tgt.arch, ext),
+			"var libddwaf []byte",
+			"",
+			fmt.Sprintf(`const embedNamePattern = "libddwaf-*.%s"`, ext),
+			"", // Trailing new line...
+		},
+		"\n",
+	)
+	if err := os.WriteFile(path.Join(libDir, fmt.Sprintf("vendor_%s_%s.go", tgt.os, tgt.arch)), []byte(gosource), 0644); err != nil {
+		panic(err)
+	}
+}
+
+func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir string, assets map[string]*github.ReleaseAsset) {
+	defer wg.Done()
+
+	tarName := fmt.Sprintf("libddwaf-%s-%s.tar.gz", version, tgt.assetLabel)
+	shaName := fmt.Sprintf("%s.sha256", tarName)
+
+	tarAsset, found := assets[tarName]
+	if !found {
+		panic(fmt.Errorf("could not find tarball named %s", tarName))
+	}
+	shaAsset, found := assets[shaName]
+	if !found {
+		panic(fmt.Errorf("could not find sha256 named %s", shaName))
+	}
+
+	tarUrl := *tarAsset.BrowserDownloadURL
+	shaUrl := *shaAsset.BrowserDownloadURL
+
+	tmpdir, err := os.MkdirTemp("", "libddwaf-*")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	if _, err := script.Get(tarUrl).WriteFile(path.Join(tmpdir, tarName)); err != nil {
+		panic(err)
+	}
+	sha, err := script.Get(shaUrl).String()
+	if err != nil {
+		panic(err)
+	}
+	sum, err := script.File(path.Join(tmpdir, tarName)).SHA256Sum()
+	if err != nil {
+		panic(err)
+	}
+	// To match the shasum format...
+	sum = fmt.Sprintf("%s  %s\n", sum, tarName)
+	if sum != sha {
+		panic(fmt.Errorf("checksum mismatch on %s:\nExpected %s\nActual   %s", tarUrl, sha, sum))
+	}
+
+	file, err := os.Open(path.Join(tmpdir, tarName))
+	if err != nil {
+		panic(err)
+	}
+	reader, err := gzip.NewReader(file)
+	if err != nil {
+		panic(err)
+	}
+	arch := gotar.NewReader(reader)
+	foundLib := false
+	foundHdr := false
+	for {
+		header, err := arch.Next()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			panic(err)
+		}
+
+		var destPath string
+		switch name := header.FileInfo().Name(); name {
+		case "libddwaf.so", "libddwaf.dylib":
+			destPath = path.Join(embedDir, name)
+			foundLib = true
+		case "ddwaf.h":
+			if tgt.primary {
+				destPath = path.Join(rootDir, "include", name)
+				foundHdr = true
+			} else {
+				continue
+			}
+		default:
+			continue
+		}
+
+		fmt.Printf("... downloaded %s\n", destPath)
+		if _, err := script.NewPipe().WithReader(arch).WriteFile(destPath); err != nil {
+			panic(err)
+		}
+
+		if foundLib && (foundHdr || !tgt.primary) {
+			break
+		}
+	}
+
+	if !foundLib {
+		panic(fmt.Errorf("could not find libddwaf.so/libddwaf.dylib in %s", tarUrl))
+	}
+	if tgt.primary && !foundHdr {
+		panic(fmt.Errorf("could not find ddwaf.h in %s", tarUrl))
+	}
+}
+
+type target struct {
+	os         string
+	arch       string
+	assetLabel string
+	primary    bool // The one we'll get ddwaf.h from
+}
+
+var targets = []target{
+	{
+		os:         "darwin",
+		arch:       "amd64",
+		assetLabel: "darwin-x86_64",
+	},
+	{
+		os:         "darwin",
+		arch:       "arm64",
+		assetLabel: "darwin-arm64",
+	},
+	{
+		os:         "linux",
+		arch:       "amd64",
+		assetLabel: "x86_64-linux-musl",
+		primary:    true,
+	},
+	{
+		os:         "linux",
+		arch:       "arm64",
+		assetLabel: "aarch64-linux-musl",
+	},
+	// These are currentlu not supported by ebitengine/purego:
+	// {os: "linux", arch: "armv7", assetLabel: "armv7-linux-musl"},
+	// {os: "linux", arch: "i386", assetLabel: "i386-linux-musl"},
+}
+
+func init() {
+	_, filename, _, _ := runtime.Caller(0)
+	dir := path.Dir(filename)
+	rootDir = path.Join(dir, "..", "..")
+	libDir = path.Join(rootDir, "internal", "vendor")
+	versionFile = path.Join(libDir, ".version")
+
+	file, err := os.Open(versionFile)
+	if errors.Is(err, os.ErrNotExist) {
+		currentVersion = "<none>"
+		return
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		panic(err)
+	}
+
+	currentVersion = string(data)
+}