Update WAF to 1.15.0

_tools/libddwaf-updater/go.mod

@@ -0,0 +1,7 @@
+module github.com/DataDog/go-libddwaf/libddwaf-updater
+
+go 1.18
+
+require github.com/google/go-github/v56 v56.0.0
+
+require github.com/google/go-querystring v1.1.0 // indirect

_tools/libddwaf-updater/go.sum

@@ -0,0 +1,7 @@
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-github/v56 v56.0.0 h1:TysL7dMa/r7wsQi44BjqlwaHvwlFlqkK8CtBWCX3gb4=
+github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

_tools/libddwaf-updater/update.go

@@ -0,0 +1,276 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+package main
+
+import (
+	gotar "archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	_ "embed"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"runtime"
+	"slices"
+	"sync"
+
+	"github.com/google/go-github/v56/github"
+)
+
+var (
+	rootDir        string
+	libDir         string
+	versionFile    string
+	currentVersion string
+)
+
+func main() {
+	force := os.Args[1] == "--force"
+
+	gh := github.NewClient(nil)
+
+	release, _, err := gh.Repositories.GetLatestRelease(context.Background(), "DataDog", "libddwaf")
+	if err != nil {
+		panic(err)
+	}
+
+	version := *release.TagName
+	if version == currentVersion {
+		fmt.Printf("Already up-to-date with v%s\n", version)
+		if force {
+			fmt.Println("--force is set, re-downloading assets anyway!")
+		} else {
+			return
+		}
+	} else {
+		fmt.Printf("Will upgrade from v%s to v%s\n", currentVersion, version)
+	}
+
+	assets := make(map[string]*github.ReleaseAsset, len(release.Assets))
+	for _, asset := range release.Assets {
+		if asset.Name == nil {
+			continue
+		}
+		assets[*asset.Name] = asset
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(len(targets))
+	for _, tgt := range targets {
+		embedDir := path.Join(libDir, tgt.embedName)
+		if _, err = os.Stat(embedDir); errors.Is(err, os.ErrNotExist) {
+			if err = os.MkdirAll(embedDir, 0755); err != nil {
+				panic(err)
+			}
+			if err = os.WriteFile(path.Join(embedDir, "vendor.go"), []byte(vendorTemplate), 0644); err != nil {
+				panic(err)
+			}
+		}
+		go handleTarget(&wg, version, tgt, embedDir, assets)
+	}
+
+	wg.Wait()
+
+	file, err := os.OpenFile(versionFile, os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		panic(err)
+	}
+	written := 0
+	for written < len(version) {
+		wrote, err := file.WriteString(version[written:])
+		if err != nil {
+			panic(err)
+		}
+		written += wrote
+	}
+
+	fmt.Println("All done! Don't forget to check in changes to include/ and lib/, check the libddwaf upgrade guide to update bindings!")
+}
+
+func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir string, assets map[string]*github.ReleaseAsset) {
+	defer wg.Done()
+
+	tarName := fmt.Sprintf("libddwaf-%s-%s.tar.gz", version, tgt.assetLabel)
+	shaName := fmt.Sprintf("%s.sha256", tarName)
+
+	tarAsset, found := assets[tarName]
+	if !found {
+		panic(fmt.Errorf("could not find tarball named %s", tarName))
+	}
+	shaAsset, found := assets[shaName]
+	if !found {
+		panic(fmt.Errorf("could not find sha256 named %s", shaName))
+	}
+
+	tarUrl := *tarAsset.BrowserDownloadURL
+	shaUrl := *shaAsset.BrowserDownloadURL
+
+	var tar []byte
+	{
+		resp, err := http.Get(tarUrl)
+		if err != nil {
+			panic(err)
+		}
+		defer resp.Body.Close()
+		tar, err = io.ReadAll(resp.Body)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	var sha string
+	{
+		resp, err := http.Get(shaUrl)
+		if err != nil {
+			panic(err)
+		}
+		defer resp.Body.Close()
+		data, err := io.ReadAll(resp.Body)
+		if err != nil {
+			panic(err)
+		}
+		split := slices.Index(data, ' ')
+		if split < 0 {
+			panic("invalid sha256 file content")
+		}
+		sha = string(data[:split])
+	}
+
+	hash := sha256.Sum256(tar)
+	sum := hex.EncodeToString(hash[:])
+
+	if sum != sha {
+		panic(fmt.Errorf("checksum mismatch on %s:\nExpected %s\nActual   %s", tarUrl, sha, sum))
+	}
+
+	reader, err := gzip.NewReader(bytes.NewReader(tar))
+	if err != nil {
+		panic(err)
+	}
+	arch := gotar.NewReader(reader)
+	foundLib := false
+	foundHdr := false
+	for {
+		header, err := arch.Next()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			panic(err)
+		}
+
+		var dest *os.File
+		switch name := header.FileInfo().Name(); name {
+		case "libddwaf.so", "libddwaf.dylib":
+			destPath := path.Join(embedDir, name)
+			fmt.Printf("... downloaded %s\n", destPath)
+			{
+				dest, err = os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+				if err != nil {
+					panic(err)
+				}
+				defer dest.Close()
+				_, err = io.Copy(dest, arch)
+			}
+
+			foundLib = true
+		case "ddwaf.h":
+			if tgt.primary {
+				destPath := path.Join(rootDir, "include", name)
+				fmt.Printf("... downloaded %s\n", destPath)
+				dest, err = os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+				if err != nil {
+					panic(err)
+				}
+				defer dest.Close()
+				_, err = io.Copy(dest, arch)
+			}
+			foundHdr = true
+		}
+		if err != nil {
+			panic(err)
+		}
+		if foundLib && foundHdr {
+			break
+		}
+	}
+
+	if !foundLib {
+		panic(fmt.Errorf("could not find libddwaf.so/libddwaf.dylib in %s", tarUrl))
+	}
+	if !foundHdr {
+		panic(fmt.Errorf("could not find ddwaf.h in %s", tarUrl))
+	}
+}
+
+type target struct {
+	embedName  string
+	assetLabel string
+	primary    bool // The one we'll get ddwaf.h from
+}
+
+var targets = []target{
+	{
+		embedName:  "darwin-amd64",
+		assetLabel: "darwin-x86_64",
+	},
+	{
+		embedName:  "darwin-arm64",
+		assetLabel: "darwin-arm64",
+	},
+	{
+		embedName:  "linux-amd64",
+		assetLabel: "x86_64-linux-musl",
+		primary:    true,
+	},
+	{
+		embedName:  "linux-arm64",
+		assetLabel: "aarch64-linux-musl",
+	},
+	{
+		embedName:  "linux-armv7",
+		assetLabel: "armv7-linux-musl",
+	},
+	{
+		embedName:  "linux-i386",
+		assetLabel: "i386-linux-musl",
+	},
+}
+
+func init() {
+	_, filename, _, _ := runtime.Caller(0)
+	dir := path.Dir(filename)
+	rootDir = path.Join(dir, "..", "..")
+	libDir = path.Join(rootDir, "lib")
+	versionFile = path.Join(libDir, ".version")
+	file, err := os.Open(versionFile)
+	if err != nil {
+		panic(err)
+	}
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		panic(err)
+	}
+
+	currentVersion = string(data)
+}
+
+const vendorTemplate = `// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+// Package vendor is required to help go tools support vendoring.
+// DO NOT REMOVE
+package vendor
+`

_tools/libddwaf-updater/update.sh

@@ -1,101 +1,4 @@
 #!/bin/bash
 
-#
-# Unless explicitly stated otherwise all files in this repository are licensed
-# under the Apache License Version 2.0.
-# This product includes software developed at Datadog (https://www.datadoghq.com/).
-# Copyright 2016 Datadog, Inc.
-#
-
-# Update the libddwaf to the latest GitHub release version.
-# Usage: ./update-libddwaf.sh
-#
-
-set -ex
-
-bindings_dir=$(readlink -f "$(dirname $0)/../../")
-
-version=""
-if [ $# -eq 1 ]; then
-    version=$1
-else
-    echo Looking up for the latest GitHub release
-    version=$(curl -s https://api.github.com/repos/DataDog/libddwaf/releases/latest | jq -r '.tag_name')
-fi
-
-echo Updating to libddwaf v$version
-
-tmpdir=$(mktemp -d /tmp/libddwaf-XXXXXXXX)
-echo Using $tmpdir
-
-LD_REQUIRED_DEFINED="--require-defined=ddwaf_init \
-                    --require-defined=ddwaf_get_version \
-                    --require-defined=ddwaf_destroy \
-                    --require-defined=ddwaf_context_init \
-                    --require-defined=ddwaf_context_destroy \
-                    --require-defined=ddwaf_required_addresses \
-                    --require-defined=ddwaf_result_free"
-
-run_binutils() {
-  docker run -it --rm -v $bindings_dir:$bindings_dir -v $tmpdir:$tmpdir -w $PWD ghcr.io/datadog/binutils-gdb:2.38 $@
-}
-
-run_strip() {
-  run_binutils $1-strip --strip-dwo --strip-unneeded --strip-debug $2
-}
-
-#
-# darwin/arm64
-#
-
-echo Updating libddwaf for darwin/arm64
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libddwaf-$version-darwin-arm64.tar.gz | tar -xz -C$tmpdir
-echo Copying the darwin/arm64 library
-cp -v "$tmpdir/libddwaf-$version-darwin-arm64/lib/libddwaf.dylib" "$bindings_dir/lib/darwin-arm64/_libddwaf.dylib"
-
-#
-# darwin/amd64
-#
-
-echo Updating libddwaf for darwin/amd64yes
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libddwaf-$version-darwin-x86_64.tar.gz | tar -xz -C$tmpdir
-echo Copying the darwin/amd64 library
-cp -v "$tmpdir/libddwaf-$version-darwin-x86_64/lib/libddwaf.dylib" "$bindings_dir/lib/darwin-amd64/_libddwaf.dylib"
-
-#
-# linux/amd64
-#
-
-echo Updating libddwaf for linux/amd64
-# 1. Download the libddwaf build
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libddwaf-$version-linux-x86_64.tar.gz | tar -xz -C$tmpdir
-# 2. Download the libc++ build
-libcxx_dir=$tmpdir/libc++-x86_64-linux
-mkdir $libcxx_dir
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libc++-static-x86_64-linux.tar.gz | tar -xz -C$libcxx_dir
-# 4. Copy & Strip
-cp -v "$tmpdir/libddwaf-$version-linux-x86_64/lib/libddwaf.so" "$bindings_dir/lib/linux-amd64/libddwaf.so"
-run_strip x86_64-linux-gnu "$bindings_dir/lib/linux-amd64/libddwaf.so"
-
-#
-# linux/arm64
-#
-
-echo Updating libddwaf for linux/arm64
-# 1. Download the libddwaf build
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libddwaf-$version-linux-aarch64.tar.gz | tar -xz -C$tmpdir
-# 2. Download the libc++ build
-libcxx_dir=$tmpdir/libc++-aarch64-linux
-mkdir $libcxx_dir
-curl -L https://github.com/DataDog/libddwaf/releases/download/$version/libc++-static-aarch64-linux.tar.gz | tar -xz -C$libcxx_dir
-# 4. Copy & Strip
-cp -v "$tmpdir/libddwaf-$version-linux-aarch64/lib/libddwaf.so" "$bindings_dir/lib/linux-arm64/libddwaf.so"
-run_strip aarch64-linux-gnu "$bindings_dir/lib/linux-arm64/libddwaf.so"
-
-#
-# ddwaf.h
-# Note that we arbitrarily take it from the linux/amd64 archive as it does not
-# depend on the target.
-#
-echo Updating ddwaf.h
-cp -v $tmpdir/libddwaf-$version-linux-x86_64/include/ddwaf.h $bindings_dir/include
+cd $(dirname $0)
+exec go run ./update.go "$@"