From 3616b8b74c3de797fb703548200d82107dcf00c4 Mon Sep 17 00:00:00 2001 From: Benjamin Arntzen Date: Sun, 24 Mar 2024 02:57:42 +0000 Subject: [PATCH] Integrate changes for new modules (#50) * Print jobID as well as dealID so we can debug Bacalhau directly * Ignore lilypad binary if built locally * Debug fix was in the wrong place * Some temporary hacks * Fix order of operations! * Add some print statements * Fix subst syntax * Add gitignore, modify stack to suit new setup --- .gitignore | 2 ++ pkg/executor/bacalhau/bacalhau.go | 11 ++++------- pkg/http/utils.go | 3 +++ pkg/module/utils.go | 11 +++++------ pkg/resourceprovider/controller.go | 5 ++++- stack | 8 +++++--- 6 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 2d72c891..8fb79842 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ hardhat/deployments/hardhat hardhat/deployments/localhost hardhat/deployments/geth hardhat/typechain-types +node_modules/* +lilypad .env *.env .env.* diff --git a/pkg/executor/bacalhau/bacalhau.go b/pkg/executor/bacalhau/bacalhau.go index a57772cc..77811f51 100644 --- a/pkg/executor/bacalhau/bacalhau.go +++ b/pkg/executor/bacalhau/bacalhau.go @@ -121,17 +121,14 @@ func (executor *BacalhauExecutor) copyJobResults(dealID string, jobID string) (s return "", fmt.Errorf("error creating a local folder of results %s -> %s", dealID, err.Error()) } - copyResultsCmd := exec.Command( - "bacalhau", - "get", - jobID, - "--output-dir", resultsDir, - ) + copyCmdText := fmt.Sprintf("bacalhau get %s --output-dir %s", jobID, resultsDir) + log.Debug().Msgf("Executing command: %s", copyCmdText) // Log the command before execution for debugging + copyResultsCmd := exec.Command("bacalhau", "get", jobID, "--output-dir", resultsDir) copyResultsCmd.Env = executor.bacalhauEnv _, err = copyResultsCmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("error copying results %s -> %s", dealID, err.Error()) + return "", fmt.Errorf("error copying results %s -> %s, command executed: %s", dealID, err.Error(), copyCmdText) } return resultsDir, nil diff --git a/pkg/http/utils.go b/pkg/http/utils.go index 9c47da22..088e3239 100644 --- a/pkg/http/utils.go +++ b/pkg/http/utils.go @@ -361,12 +361,15 @@ func PostRequestBuffer[ResultType any]( defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { + // Print a debug line with the response body. + log.Debug().Msgf("[debug] error while reading. response body: %s", body) return result, err } // parse body as json into result err = json.Unmarshal(body, &result) if err != nil { + log.Debug().Msgf("[debug] error while unmarshaling. response body: %s", body) return result, err } diff --git a/pkg/module/utils.go b/pkg/module/utils.go index eb49e1a0..98306ac7 100644 --- a/pkg/module/utils.go +++ b/pkg/module/utils.go @@ -189,8 +189,7 @@ func PrepareModule(module data.ModuleConfig) (string, error) { } func subst(format string, jsonEncodedInputs ...string) string { - - jsonDecodedInputs := make([]any, 0, len(jsonEncodedInputs)) + jsonDecodedInputs := make([]interface{}, 0, len(jsonEncodedInputs)) for _, input := range jsonEncodedInputs { var s string @@ -202,7 +201,7 @@ func subst(format string, jsonEncodedInputs ...string) string { jsonDecodedInputs = append(jsonDecodedInputs, s) } - log.Printf("jsonDecodedInputs:%v", jsonDecodedInputs) + log.Printf("jsonDecodedInputs: %v", jsonDecodedInputs) return fmt.Sprintf(format, jsonDecodedInputs...) } @@ -218,12 +217,12 @@ func LoadModule(module data.ModuleConfig, inputs map[string]string) (*data.Modul // TODO: golang handlebars implementation, with shortcode for string encoding e.g. escape_string templateName := fmt.Sprintf("%s-%s-%s", module.Repo, module.Path, module.Hash) - tmpl, err := template.New(templateName).Parse(moduleText) - tmpl.Funcs(template.FuncMap{ + tmpl := template.New(templateName).Funcs(template.FuncMap{ "subst": subst, }) + tmpl, err = tmpl.Parse(moduleText) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse template: %v", err) } newInputs := make(map[string]string) diff --git a/pkg/resourceprovider/controller.go b/pkg/resourceprovider/controller.go index e99176b5..a653f584 100644 --- a/pkg/resourceprovider/controller.go +++ b/pkg/resourceprovider/controller.go @@ -425,8 +425,11 @@ func (controller *ResourceProviderController) runJob(deal data.DealContainer) { controller.log.Info(fmt.Sprintf("uploading results: %s %s %s", deal.ID, executorResult.ResultsDir, executorResult.ResultsCID), executorResult.ResultsDir) - _, err = controller.solverClient.UploadResultFiles(deal.ID, executorResult.ResultsDir) + response, err := controller.solverClient.UploadResultFiles(deal.ID, executorResult.ResultsDir) + if err != nil { + // Log the response body in debug mode + controller.log.Debug("[debug] error uploading results. response was ", response) return fmt.Errorf("error uploading results: %s", err.Error()) } diff --git a/stack b/stack index 71d310df..3cfac904 100755 --- a/stack +++ b/stack @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export DEFAULT_DATA_DIR="/tmp/geth" +export DEFAULT_DATA_DIR="/data/geth" export DATA_DIRECTORY=${DATA_DIRECTORY:="$DEFAULT_DATA_DIR"} export NETWORK=${NETWORK:="geth"} @@ -27,7 +27,7 @@ function geth() { -p 8545:8545 \ -p 8546:8546 \ -v ${DATA_DIRECTORY}:/data/geth \ - ethereum/client-go \ + ethereum/client-go:v1.13.5 \ --datadir /data/geth \ --dev \ --ws \ @@ -301,7 +301,7 @@ function solver() { export JOB_CREATOR_ADDRESS=$JOB_CREATOR_ADDRESS export SERVICE_MEDIATORS=$MEDIATOR_ADDRESS export SERVER_PORT=8080 - export SERVER_URL=http://localhost:8080 + export SERVER_URL=http://testnet.lilypad.tech:8080 go run . solver "$@" } @@ -318,6 +318,7 @@ function jobcreator() { function resource-provider() { source .env eval $(./stack print-local-dev-env) + export WEB3_RPC_URL=ws://testnet.lilypad.tech:8546 export WEB3_PRIVATE_KEY=$RESOURCE_PROVIDER_PRIVATE_KEY export SERVICE_SOLVER=$SOLVER_ADDRESS export SERVICE_MEDIATORS=$MEDIATOR_ADDRESS @@ -327,6 +328,7 @@ function resource-provider() { function mediator() { source .env eval $(./stack print-local-dev-env) + export WEB3_RPC_URL=ws://testnet.lilypad.tech:8546 export WEB3_PRIVATE_KEY=$MEDIATOR_PRIVATE_KEY export WEB3_DIRECTORY_ADDRESS=$DIRECTORY_ADDRESS export SERVICE_SOLVER=$SOLVER_ADDRESS