feat: Add run job traces (#292)

* refactor: Move version and sha to system package

We want to include this information in traces so time to move it out of
the command.

* feat: Add telemetry URL config

* feat: Add tracing provider

* feat: Add run job trace

* refactor: Move away from otel global provider registration

We were unable to access the global provider in the resource provider
package. Somehow the registration didn't hold, and we were getting a
seemingly noop provider.

* chore: Add OTel service name helper

* chore: Debug log public address at init

* chore: Redact private key in new contract SDK log

* feat: Expand run job trace

* chore: Upgrade to Go 1.22

* feat: Add system and chain resource attributes

* chore: Debug log job offer ID in jobcreator run

* feat: Add resource GPU info

Co-authored-by: logan <[email protected]>

* chore: Update telemetry port

* feat: Add auth and https config to exporter

* refactor: Separate out telemetry options

* feat: Add disable telemetry flag

* build: Disable telemetry in test workflow

* chore: Remove cached hardhat openzeppelin files

* chore: Set disable telemetry for stack compose-env

* chore: Fix GPU info nil pointer exceptions

* chore: Improve logs

* build: Add debug logs

* chore: Debug log trace ID

* chore: Log deal ID

* chore: Update local dev telemetry URL port

---------

Co-authored-by: logan <[email protected]>
Co-authored-by: Narb <[email protected]>

.github/workflows/release.yml

@@ -170,7 +170,7 @@ jobs:
           echo "Building for ${GOOS}/${GOARCH} with GOOS=$GOOS, GOARCH=$GOARCH"
           echo "Excluding CUDA. Use the 'cuda' build tag to include it."
 
-          go build -o "build/lilypad-${GOOS}-${GOARCH}-cpu" -v -ldflags="-X 'github.com/lilypad-tech/lilypad/cmd/lilypad.version=${{ needs.release.outputs.tag_name }}' -X 'github.com/lilypad-tech/lilypad/cmd/lilypad.commitSHA=${{ needs.release.outputs.sha }}'"
+          go build -o "build/lilypad-${GOOS}-${GOARCH}-cpu" -v -ldflags="-X 'github.com/lilypad-tech/lilypad/pkg/system.Version=${{ needs.release.outputs.tag_name }}' -X 'github.com/lilypad-tech/lilypad/pkg/system.CommitSHA=${{ needs.release.outputs.sha }}'"
 
           echo "-------------- OS: ${GOOS} : Arch: ${GOARCH} ---------- done"
 
@@ -206,7 +206,7 @@ jobs:
           echo "CUDA_HOME: $CUDA_HOME"
           echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
 
-          go build -o "build/lilypad-${GOOS}-${GOARCH}-gpu" -v -tags cuda -ldflags="-X 'github.com/lilypad-tech/lilypad/cmd/lilypad.version=${{ needs.release.outputs.tag_name }}' -X 'github.com/lilypad-tech/lilypad/cmd/lilypad.commitSHA=${{ needs.release.outputs.sha }}'"
+          go build -o "build/lilypad-${GOOS}-${GOARCH}-gpu" -v -tags cuda -ldflags="-X 'github.com/lilypad-tech/lilypad/pkg/system.Version=${{ needs.release.outputs.tag_name }}' -X 'github.com/lilypad-tech/lilypad/pkg/system.CommitSHA=${{ needs.release.outputs.sha }}'"
 
           echo "-------------- OS: ${GOOS} : Arch: ${GOARCH} ---------- done"
 

.github/workflows/test.yml

@@ -27,7 +27,20 @@ jobs:
         run: ./stack compose-init
 
       - name: Run stack
+        env:
+          DISABLE_TELEMETRY: true
         run: ./stack compose-up -d
 
       - name: Run tests
+        env:
+          LOG_LEVEL: debug
         run: ./stack integration-tests
+
+      - name: Display resource provider logs
+        run: docker logs resource-provider
+
+      - name: Display solver logs
+        run: docker logs solver
+
+      - name: Display chain logs
+        run: docker logs chain

cmd/lilypad/resource-provider.go

@@ -1,6 +1,8 @@
 package lilypad
 
 import (
+	"fmt"
+
 	"github.com/lilypad-tech/lilypad/pkg/executor/bacalhau"
 	optionsfactory "github.com/lilypad-tech/lilypad/pkg/options"
 	"github.com/lilypad-tech/lilypad/pkg/resourceprovider"
@@ -24,7 +26,7 @@ func newResourceProviderCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			return runResourceProvider(cmd, options)
+			return runResourceProvider(cmd, options, network)
 		},
 	}
 
@@ -33,7 +35,7 @@ func newResourceProviderCmd() *cobra.Command {
 	return resourceProviderCmd
 }
 
-func runResourceProvider(cmd *cobra.Command, options resourceprovider.ResourceProviderOptions) error {
+func runResourceProvider(cmd *cobra.Command, options resourceprovider.ResourceProviderOptions, network string) error {
 	commandCtx := system.NewCommandContext(cmd)
 	defer commandCtx.Cleanup()
 
@@ -47,7 +49,22 @@ func runResourceProvider(cmd *cobra.Command, options resourceprovider.ResourcePr
 		return err
 	}
 
-	resourceProviderService, err := resourceprovider.NewResourceProvider(options, web3SDK, executor)
+	tc := system.TelemetryConfig{
+		TelemetryURL:   options.Telemetry.URL,
+		TelemetryToken: options.Telemetry.Token,
+		Enabled:        !options.Telemetry.Disable,
+		Service:        system.ResourceProviderService,
+		Network:        network,
+		Address:        web3SDK.GetAddress().String(),
+		GPU:            system.GetGPUInfo(),
+	}
+	telemetry, err := system.SetupOTelSDK(commandCtx.Ctx, tc)
+	if err != nil {
+		fmt.Printf("failed to setup opentelemetry: %s", err)
+	}
+	commandCtx.Cm.RegisterCallbackWithContext(telemetry.Shutdown)
+
+	resourceProviderService, err := resourceprovider.NewResourceProvider(options, web3SDK, executor, telemetry)
 	if err != nil {
 		return err
 	}

cmd/lilypad/root.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/lilypad-tech/lilypad/pkg/system"
 	"github.com/spf13/cobra"
 )
 
@@ -19,7 +20,7 @@ func NewRootCmd() *cobra.Command {
 	RootCmd := &cobra.Command{
 		Use:   getCommandLineExecutable(),
 		Short: "Lilypad",
-		Long:  fmt.Sprintf("Lilypad: %s \nCommit: %s \n", version, commitSHA),
+		Long:  fmt.Sprintf("Lilypad: %s \nCommit: %s \n", system.Version, system.CommitSHA),
 	}
 
 	var network string

cmd/lilypad/run.go

@@ -46,18 +46,18 @@ func runJob(cmd *cobra.Command, options jobcreator.JobCreatorOptions) error {
 	c := color.New(color.FgCyan).Add(color.Bold)
 	header := `
 ⠀⠀⠀⠀⠀⠀⣀⣤⣤⢠⣤⣀⠀⠀⠀⠀⠀
-⠀⠀⠀⠀⢴⣿⣿⣿⣿⢸⣿⡟⠀⠀⠀⠀⠀    ██╗     ██╗██╗  ██╗   ██╗██████╗  █████╗ ██████╗ 
+⠀⠀⠀⠀⢴⣿⣿⣿⣿⢸⣿⡟⠀⠀⠀⠀⠀    ██╗     ██╗██╗  ██╗   ██╗██████╗  █████╗ ██████╗
 ⠀⠀⣰⣿⣦⡙⢿⣿⣿⢸⡿⠀⠀⠀⠀⢀⠀    ██║     ██║██║  ╚██╗ ██╔╝██╔══██╗██╔══██╗██╔══██╗
 ⠀⢰⣿⣿⣿⣿⣦⡙⣿⢸⠁⢀⣠⣴⣾⣿⡆    ██║     ██║██║   ╚████╔╝ ██████╔╝███████║██║  ██║
 ⠀⣛⣛⣛⣛⣛⣛⣛⠈⠀⣚⣛⣛⣛⣛⣛⣛    ██║     ██║██║    ╚██╔╝  ██╔═══╝ ██╔══██║██║  ██║
 ⠀⢹⣿⣿⣿⣿⠟⣡⣿⢸⣮⡻⣿⣿⣿⣿⡏    ███████╗██║███████╗██║   ██║     ██║  ██║██████╔╝
 ⠀⠀⢻⣿⡟⣩⣾⣿⣿⢸⣿⣿⣌⠻⣿⡟⠀    ╚══════╝╚═╝╚══════╝╚═╝   ╚═╝     ╚═╝  ╚═╝╚═════╝ v2
-⠀⠀⠀⠉⢾⣿⣿⣿⣿⢸⣿⣿⣿⡷⠈⠀⠀                                                  
+⠀⠀⠀⠉⢾⣿⣿⣿⣿⢸⣿⣿⣿⡷⠈⠀⠀
 ⠀⠀⠀⠀⠀⠈⠙⠛⠛⠘⠛⠋⠁⠀ ⠀⠀⠀   Decentralized Compute Network  https://lilypad.tech
 
 `
-	if version != "" {
-		header = strings.Replace(header, "v2", version, 1)
+	if system.Version != "" {
+		header = strings.Replace(header, "v2", system.Version, 1)
 	}
 	c.Print(header)
 
@@ -92,6 +92,7 @@ func runJob(cmd *cobra.Command, options jobcreator.JobCreatorOptions) error {
 		fmt.Println(err.Error())
 		os.Exit(1)
 	}
+
 	commandCtx := system.NewCommandContext(cmd)
 	defer commandCtx.Cleanup()
 	result, err := jobcreator.RunJob(commandCtx, options, func(evOffer data.JobOfferContainer) {

cmd/lilypad/version.go

@@ -9,9 +9,6 @@ import (
 	"github.com/lilypad-tech/lilypad/pkg/system"
 )
 
-var version string
-var commitSHA string
-
 const goBinaryURL = "https://github.com/lilypad-tech/lilypad/releases/"
 
 func newVersionCmd() *cobra.Command {
@@ -36,14 +33,14 @@ func runVersion(cmd *cobra.Command) error {
 	commandCtx := system.NewCommandContext(cmd)
 	defer commandCtx.Cleanup()
 
-	if version == "" {
+	if system.Version == "" {
 		fmt.Printf("version not found: download the latest binary from %s", goBinaryURL)
 		// unnecessary help shows up when returned as error, so shortciruting here
 		return nil
 	}
 
-	fmt.Printf("Lilypad: %s\n", version)
-	fmt.Printf("Commit: %s\n", commitSHA)
+	fmt.Printf("Lilypad: %s\n", system.Version)
+	fmt.Printf("Commit: %s\n", system.CommitSHA)
 
 	// TODO: suggest updating to the latest version if the current version is not the latest version
 

docker/docker-compose.dev.yml

@@ -101,6 +101,7 @@ services:
       - WEB3_PRIVATE_KEY=${RESOURCE_PROVIDER_PRIVATE_KEY}
       - BACALHAU_SERVE_IPFS_PATH=/tmp/lilypad/data/ipfs
       - LOG_LEVEL=debug
+      - DISABLE_TELEMETRY=${DISABLE_TELEMETRY}
 volumes:
   chain-data:
   bacalhau-data:

docker/resource-provider/Dockerfile

@@ -8,6 +8,7 @@ ARG NETWORK=testnet
 ENV LOG_LEVEL=info
 ENV BACALHAU_API_HOST="localhost"
 ENV WEB3_PRIVATE_KEY=""
+ENV DISABLE_TELEMETRY=false
 
 # Install necessary dependencies
 RUN apt update && apt install -y wget bash && apt clean
@@ -46,7 +47,7 @@ RUN echo "#!/bin/bash" >> run
 # Launch Bacalhau
 RUN echo "/usr/local/bin/bacalhau serve --node-type compute,requester --peer none --private-internal-ipfs=false --job-selection-accept-networked &" >> run
 # Launch Lilypad
-RUN echo "/usr/local/bin/lilypad resource-provider --network ${NETWORK} --disable-pow ${DISABLE_POW} &" >> run
+RUN echo "/usr/local/bin/lilypad resource-provider --network ${NETWORK} --disable-pow=${DISABLE_POW} --disable-telemetry=${DISABLE_TELEMETRY} &" >> run
 RUN echo "wait -n" >> run
 RUN chmod +x run
 

go.mod

@@ -1,14 +1,14 @@
 module github.com/lilypad-tech/lilypad
 
-go 1.20
+go 1.22
 
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/davecgh/go-spew v1.1.1
 	github.com/ethereum/go-ethereum v1.13.4
 	github.com/fatih/color v1.15.0
 	github.com/go-git/go-git/v5 v5.10.0
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
@@ -18,7 +18,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/cobra v1.7.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/theckman/yacspin v0.13.12
 	gorgonia.org/cu v0.9.7-0.20240623234718-3cd40db700e9
 	k8s.io/apimachinery v0.28.3
@@ -32,6 +32,7 @@ require (
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/bits-and-blooms/bitset v1.7.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
@@ -42,14 +43,16 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ethereum/c-kzg-4844 v0.3.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -70,6 +73,8 @@ require (
 	github.com/ipfs/go-verifcid v0.0.1 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-prime v0.20.0 // indirect
+	github.com/jaypipes/ghw v0.12.0 // indirect
+	github.com/jaypipes/pcidb v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -78,6 +83,7 @@ require (
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.0.4 // indirect
@@ -99,21 +105,34 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	go.opentelemetry.io/otel v1.7.0 // indirect
-	go.opentelemetry.io/otel/trace v1.7.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.22.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
+	google.golang.org/grpc v1.64.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	howett.net/plist v1.0.0 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )