Integrate changes for new modules (#50)

* Print jobID as well as dealID so we can debug Bacalhau directly

* Ignore lilypad binary if built locally

* Debug fix was in the wrong place

* Some temporary hacks

* Fix order of operations!

* Add some print statements

* Fix subst syntax

* Add gitignore, modify stack to suit new setup

.gitignore

@@ -5,6 +5,8 @@ hardhat/deployments/hardhat
 hardhat/deployments/localhost
 hardhat/deployments/geth
 hardhat/typechain-types
+node_modules/*
+lilypad
 .env
 *.env
 .env.*

pkg/executor/bacalhau/bacalhau.go

@@ -121,17 +121,14 @@ func (executor *BacalhauExecutor) copyJobResults(dealID string, jobID string) (s
 		return "", fmt.Errorf("error creating a local folder of results %s -> %s", dealID, err.Error())
 	}
 
-	copyResultsCmd := exec.Command(
-		"bacalhau",
-		"get",
-		jobID,
-		"--output-dir", resultsDir,
-	)
+	copyCmdText := fmt.Sprintf("bacalhau get %s --output-dir %s", jobID, resultsDir)
+	log.Debug().Msgf("Executing command: %s", copyCmdText) // Log the command before execution for debugging
+	copyResultsCmd := exec.Command("bacalhau", "get", jobID, "--output-dir", resultsDir)
 	copyResultsCmd.Env = executor.bacalhauEnv
 
 	_, err = copyResultsCmd.CombinedOutput()
 	if err != nil {
-		return "", fmt.Errorf("error copying results %s -> %s", dealID, err.Error())
+		return "", fmt.Errorf("error copying results %s -> %s, command executed: %s", dealID, err.Error(), copyCmdText)
 	}
 
 	return resultsDir, nil

pkg/http/utils.go

@@ -361,12 +361,15 @@ func PostRequestBuffer[ResultType any](
 	defer resp.Body.Close()
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		// Print a debug line with the response body.
+		log.Debug().Msgf("[debug] error while reading. response body: %s", body)
 		return result, err
 	}
 
 	// parse body as json into result
 	err = json.Unmarshal(body, &result)
 	if err != nil {
+		log.Debug().Msgf("[debug] error while unmarshaling. response body: %s", body)
 		return result, err
 	}
 

pkg/module/utils.go

@@ -189,8 +189,7 @@ func PrepareModule(module data.ModuleConfig) (string, error) {
 }
 
 func subst(format string, jsonEncodedInputs ...string) string {
-
-	jsonDecodedInputs := make([]any, 0, len(jsonEncodedInputs))
+	jsonDecodedInputs := make([]interface{}, 0, len(jsonEncodedInputs))
 
 	for _, input := range jsonEncodedInputs {
 		var s string
@@ -202,7 +201,7 @@ func subst(format string, jsonEncodedInputs ...string) string {
 
 		jsonDecodedInputs = append(jsonDecodedInputs, s)
 	}
-	log.Printf("jsonDecodedInputs:%v", jsonDecodedInputs)
+	log.Printf("jsonDecodedInputs: %v", jsonDecodedInputs)
 
 	return fmt.Sprintf(format, jsonDecodedInputs...)
 }
@@ -218,12 +217,12 @@ func LoadModule(module data.ModuleConfig, inputs map[string]string) (*data.Modul
 	// TODO: golang handlebars implementation, with shortcode for string encoding e.g. escape_string
 
 	templateName := fmt.Sprintf("%s-%s-%s", module.Repo, module.Path, module.Hash)
-	tmpl, err := template.New(templateName).Parse(moduleText)
-	tmpl.Funcs(template.FuncMap{
+	tmpl := template.New(templateName).Funcs(template.FuncMap{
 		"subst": subst,
 	})
+	tmpl, err = tmpl.Parse(moduleText)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to parse template: %v", err)
 	}
 
 	newInputs := make(map[string]string)

pkg/resourceprovider/controller.go

@@ -425,8 +425,11 @@ func (controller *ResourceProviderController) runJob(deal data.DealContainer) {
 
 		controller.log.Info(fmt.Sprintf("uploading results: %s %s %s", deal.ID, executorResult.ResultsDir, executorResult.ResultsCID), executorResult.ResultsDir)
 
-		_, err = controller.solverClient.UploadResultFiles(deal.ID, executorResult.ResultsDir)
+		response, err := controller.solverClient.UploadResultFiles(deal.ID, executorResult.ResultsDir)
+
 		if err != nil {
+			// Log the response body in debug mode
+			controller.log.Debug("[debug] error uploading results. response was ", response)
 			return fmt.Errorf("error uploading results: %s", err.Error())
 		}
 

stack

@@ -3,7 +3,7 @@ set -euo pipefail
 IFS=$'\n\t'
 
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export DEFAULT_DATA_DIR="/tmp/geth"
+export DEFAULT_DATA_DIR="/data/geth"
 export DATA_DIRECTORY=${DATA_DIRECTORY:="$DEFAULT_DATA_DIR"}
 export NETWORK=${NETWORK:="geth"}
 
@@ -27,7 +27,7 @@ function geth() {
     -p 8545:8545 \
     -p 8546:8546 \
     -v ${DATA_DIRECTORY}:/data/geth \
-    ethereum/client-go \
+    ethereum/client-go:v1.13.5 \
       --datadir /data/geth \
       --dev \
       --ws \
@@ -301,7 +301,7 @@ function solver() {
   export JOB_CREATOR_ADDRESS=$JOB_CREATOR_ADDRESS
   export SERVICE_MEDIATORS=$MEDIATOR_ADDRESS
   export SERVER_PORT=8080
-  export SERVER_URL=http://localhost:8080
+  export SERVER_URL=http://testnet.lilypad.tech:8080
   go run . solver "$@"
 }
 
@@ -318,6 +318,7 @@ function jobcreator() {
 function resource-provider() {
   source .env
   eval $(./stack print-local-dev-env)
+  export WEB3_RPC_URL=ws://testnet.lilypad.tech:8546
   export WEB3_PRIVATE_KEY=$RESOURCE_PROVIDER_PRIVATE_KEY
   export SERVICE_SOLVER=$SOLVER_ADDRESS
   export SERVICE_MEDIATORS=$MEDIATOR_ADDRESS
@@ -327,6 +328,7 @@ function resource-provider() {
 function mediator() {
   source .env
   eval $(./stack print-local-dev-env)
+  export WEB3_RPC_URL=ws://testnet.lilypad.tech:8546
   export WEB3_PRIVATE_KEY=$MEDIATOR_PRIVATE_KEY
   export WEB3_DIRECTORY_ADDRESS=$DIRECTORY_ADDRESS
   export SERVICE_SOLVER=$SOLVER_ADDRESS