Skip to content

Commit

Permalink
feat: use config-file on CDK for kurtosis e2e tests (#99)
Browse files Browse the repository at this point in the history
- The e2e test for CDK use a local configuration template instead of the one on kurtosis branch: to allow to develop changes on config file is more easy if we are able to use the config file on CDK repo. After that we can update config file and cdk image on kurtosis repo
- The local configuration for run on vscode use the same template (`test/config/kurtosis-cdk-node-config.toml.template`)  as the e2e test
- Fix error on CI `test-e2e.yml` with the duplicated `ref` key
  • Loading branch information
joanestebanr authored Oct 1, 2024
1 parent ba2b077 commit 8167397
Show file tree
Hide file tree
Showing 7 changed files with 467 additions and 209 deletions.
1 change: 0 additions & 1 deletion .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ jobs:
uses: actions/checkout@v4
with:
repository: 0xPolygon/kurtosis-cdk
ref: fix/missing_cdk_config_rollupmanager
path: "kurtosis-cdk"
ref: "v0.2.11"

Expand Down
284 changes: 227 additions & 57 deletions scripts/local_config
Original file line number Diff line number Diff line change
@@ -1,30 +1,180 @@
#!/bin/bash
#Include common varaibles
source $(dirname $0)/../test/scripts/env.sh
###############################################################################
function log_debug() {
echo -e "\033[0;30mDebug: $*" "\033[0m"
}
###############################################################################
function log_error() {
echo -e "\033[0;31mError: $*" "\033[0m"
}
###############################################################################
function log_fatal() {
log_error $*
exit 1
}
###############################################################################
function ok_or_fatal(){
if [ $? -ne 0 ]; then
log_fatal $*
fi
}

###############################################################################
function get_value_from_toml_file(){
local _FILE="$1"
# KEY = <section1>.<section2>
local _SECTION="$2"
local _KEY="$3"
local _LINE
local _inside_section=0
while read -r _LINE; do
# Clean up line from spaces and tabs
_LINE=$(echo $_LINE | tr -d '[:space:]')
#echo $_LINE
if [ $_inside_section -eq 1 ]; then
if [[ "$_LINE" == [* ]]; then
return 1
fi
#local _key_splitted=(${_LINE//=/ })
local _key_name=$(echo $_LINE | cut -f 1 -d "=")
local _key_value=$(echo $_LINE | cut -f 2- -d "=")
if [ "$_key_name" == "$_KEY" ]; then
echo $_key_value
return 0
fi
elif [ "$_LINE" == "[${_SECTION}]" ]; then
_inside_section=1
fi


done < "$_FILE"
return 2

}
###############################################################################
function export_key_from_toml_file_or_fatal(){
local _EXPORTED_VAR_NAME="$1"
local _FILE="$2"
local _SECTION="$3"
local _KEY="$4"
local _VALUE=$(get_value_from_toml_file $_FILE $_SECTION $_KEY)
if [ -z "$_VALUE" ]; then
log_fatal "$FUNCNAME: key $_KEY not found in section $_SECTION"
fi
export $_EXPORTED_VAR_NAME="$_VALUE"
log_debug "$_EXPORTED_VAR_NAME=${!_EXPORTED_VAR_NAME} \t\t\t# file:$_FILE section:$_SECTION key:$_KEY"
}

###############################################################################
function export_obj_key_from_toml_file_or_fatal(){
local _EXPORTED_VAR_NAME="$1"
local _FILE="$2"
local _SECTION="$3"
local _KEY="$4"
local _OBJ_KEY="$5"
local _VALUE=$(get_value_from_toml_file $_FILE $_SECTION $_KEY)
if [ -z "$_VALUE" ]; then
log_fatal "$FUNCNAME: obj_key $_KEY not found in section $_SECTION"
fi
local _CLEAN_VALUE=$(echo $_VALUE | tr -d '{' | tr -d '}' | tr ',' '\n')
while read -r _LINE; do
local _key_splitted=(${_LINE//=/ })
if [ "${_key_splitted[0]}" == "$_OBJ_KEY" ]; then
export $_EXPORTED_VAR_NAME=${_key_splitted[1]}
log_debug "$_EXPORTED_VAR_NAME=${!_EXPORTED_VAR_NAME} \t\t\t# file:$_FILE section:$_SECTION key:$_KEY obj_key:$_OBJ_KEY"
return 0
fi
done <<< "$_CLEAN_VALUE"
log_fatal "obj_key $_OBJ_KEY not found in section $_SECTION/ $_KEY = $_VALUE"
}

###############################################################################
function export_values_of_genesis(){
local _GENESIS_FILE=$1
if [ ! -f $_GENESIS_FILE ]; then
echo "Error: genesis file not found: $_GENESIS_FILE"
exit 1
log_fatal "Error: genesis file not found: $_GENESIS_FILE"
fi
export l1_chain_id=$(jq -r '.L1Config.chainId' $_GENESIS_FILE | tr -d '"')
export pol_token_address=$(jq -r '.L1Config.polTokenAddress' $_GENESIS_FILE)
export zkevm_rollup_address=$(jq -r '.L1Config.polygonZkEVMAddress' $_GENESIS_FILE)
export zkevm_rollup_manager_address=$(jq -r '.L1Config.polygonRollupManagerAddress' $_GENESIS_FILE)
export zkevm_global_exit_root_address=$(jq -r '.L1Config.polygonZkEVMGlobalExitRootAddress' $_GENESIS_FILE)
export zkevm_rollup_manager_block_number=$(jq -r '.rollupManagerCreationBlockNumber' $_GENESIS_FILE)
}

###############################################################################
function export_values_of_cdk_node_config(){
local _CDK_CONFIG_FILE=$1
export_key_from_toml_file_or_fatal zkevm_l2_sequencer_address $_CDK_CONFIG_FILE SequenceSender L2Coinbase
export_obj_key_from_toml_file_or_fatal zkevm_l2_keystore_password $_CDK_CONFIG_FILE SequenceSender PrivateKey Password
export_key_from_toml_file_or_fatal l1_chain_id $_CDK_CONFIG_FILE SequenceSender.EthTxManager.Etherman L1ChainID
export_key_from_toml_file_or_fatal zkevm_is_validium $_CDK_CONFIG_FILE Common IsValidiumMode
export_key_from_toml_file_or_fatal zkevm_contract_versions $_CDK_CONFIG_FILE Common ContractVersions
export_key_from_toml_file_or_fatal l2_chain_id $_CDK_CONFIG_FILE Aggregator ChainID
export_key_from_toml_file_or_fatal zkevm_aggregator_port $_CDK_CONFIG_FILE Aggregator Port
export_key_from_toml_file_or_fatal zkevm_l2_agglayer_address $_CDK_CONFIG_FILE Aggregator SenderAddress
export_key_from_toml_file_or_fatal aggregator_db_name $_CDK_CONFIG_FILE Aggregator.DB Name
export_key_from_toml_file_or_fatal aggregator_db_user $_CDK_CONFIG_FILE Aggregator.DB User
export_key_from_toml_file_or_fatal aggregator_db_password $_CDK_CONFIG_FILE Aggregator.DB Password
export_key_from_toml_file_or_fatal zkevm_rollup_fork_id $_CDK_CONFIG_FILE Aggregator ForkId
export is_cdk_validium=$zkevm_is_validium
export zkevm_rollup_chain_id=$l2_chain_id


if [ "$zkevm_is_validium" == "true" ]; then
log_debug "Validium mode detected... Retrieving the dac_port"
export_value_from_kurtosis_or_fail dac_port zkevm-dac-001 dac
fi
}
###############################################################################
# MAIN
function export_value_from_kurtosis_or_fail(){
local _EXPORTED_VAR_NAME="$1"
local _SERVICE="$2"
local _END_POINT="$3"
export $_EXPORTED_VAR_NAME=$(kurtosis port print $KURTOSIS_ENCLAVE $_SERVICE $_END_POINT)
if [ -z $_EXPORTED_VAR_NAME ]; then
log_fatal "Error getting kurtosis port: $KURTOSIS_ENCLAVE $_SERVICE $_END_POINT"
fi
log_debug "$_EXPORTED_VAR_NAME=${!_EXPORTED_VAR_NAME} \t\t\t# Kurtosis $KURTOSIS_ENCLAVE $_SERVICE $_END_POINT"
}
###############################################################################
set -o pipefail # enable strict command pipe error detection
function export_portnum_from_kurtosis_or_fail(){
local _EXPORTED_VAR_NAME="$1"
export_value_from_kurtosis_or_fail "$1" "$2" "$3" > /dev/null
local _VALUE
eval "_VALUE=\$$1"
local _PORT=$(echo "$_VALUE" | cut -f 3 -d ":")
if [ -z $_PORT ]; then
log_fatal "Error getting port number from kurtosis: $2 $3 -> $_VALUE"
fi
export $_EXPORTED_VAR_NAME=$_PORT
log_debug "$_EXPORTED_VAR_NAME=${!_EXPORTED_VAR_NAME} \t\t\t# Kurtosis $KURTOSIS_ENCLAVE $2 $3"
}
###############################################################################
function export_ports_from_kurtosis(){
export_portnum_from_kurtosis_or_fail l1_rpc_port el-1-geth-lighthouse rpc
export_value_from_kurtosis_or_fail l1_rpc_url el-1-geth-lighthouse rpc
export_value_from_kurtosis_or_fail l2_rpc_url cdk-erigon-node-001 http-rpc
export_portnum_from_kurtosis_or_fail zkevm_rpc_http_port cdk-erigon-node-001 http-rpc
export_portnum_from_kurtosis_or_fail zkevm_data_streamer_port cdk-erigon-sequencer-001 data-streamer
export_portnum_from_kurtosis_or_fail aggregator_db_port postgres-001 postgres
export_portnum_from_kurtosis_or_fail agglayer_port agglayer agglayer
export aggregator_db_hostname="127.0.0.1"
}

which kurtosis > /dev/null
if [ $? -ne 0 ]; then
echo "kurtosis is not installed. Please install it:"
###############################################################################
function export_forced_values(){
export global_log_level="debug"
export l2_rpc_name="localhost"
export sequencer_name="localhost"
export deployment_suffix=""
}
###############################################################################
function check_requirements(){
which kurtosis > /dev/null
if [ $? -ne 0 ]; then
log_error "kurtosis is not installed. Please install it:"
cat << EOF
echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
Expand All @@ -33,58 +183,78 @@ if [ $? -ne 0 ]; then
EOF
exit 1

fi
fi
if [ -z $TMP_CDK_FOLDER -o -z $KURTOSIS_ENCLAVE ]; then
log_fatal "TMP_CDK_FOLDER or KURTOSIS_ENCLAVE is not set. Must be set on file env.sh"
fi
kurtosis enclave inspect $KURTOSIS_ENCLAVE > /dev/null
if [ $? -ne 0 ]; then
log_error "Error inspecting enclave $KURTOSIS_ENCLAVE"
echo "You must start kurtosis environment before running this script"
echo "- start kurtosis:"
echo " kurtosis clean --all; kurtosis run --enclave $KURTOSIS_ENCLAVE --args-file params.yml --image-download always ."

if [ -z $TMP_CDK_FOLDER -o -z $KURTOSIS_ENCLAVE ]; then
echo "TMP_CDK_FOLDER or KURTOSIS_ENCLAVE is not set. Must be set on file env.sh"
exit 1
fi
kurtosis enclave inspect $KURTOSIS_ENCLAVE > /dev/null
if [ $? -ne 0 ]; then
echo "Error inspecting enclave $KURTOSIS_ENCLAVE"
echo "You must start kurtosis environment before running this script"
echo "- start kurtosis:"
echo " kurtosis clean --all; kurtosis run --enclave $KURTOSIS_ENCLAVE --args-file params.yml --image-download always ."
exit 1
fi
}
###############################################################################
function create_dest_folder(){
export DEST=${TMP_CDK_FOLDER}/local_config
[ ! -d ${DEST} ] && mkdir -p ${DEST}
rm $DEST/*
}
###############################################################################
function download_kurtosis_artifacts(){
kurtosis files download $KURTOSIS_ENCLAVE genesis $DEST
ok_or_fatal "Error downloading kurtosis artifact genesis to $DEST"
export genesis_file=$DEST/genesis.json

kurtosis files download $KURTOSIS_ENCLAVE sequencer-keystore $DEST
ok_or_fatal "Error downloading kurtosis artifact sequencer-keystore to $DEST"
export sequencer_keystore_file=$DEST/sequencer.keystore

kurtosis files download $KURTOSIS_ENCLAVE cdk-node-config-artifact $DEST
ok_or_fatal "Error downloading kurtosis artifact cdk-node-config-artifact to $DEST"
}
###############################################################################
function check_generated_config_file(){
grep "<no value>" $DEST_TEMPLATE_FILE > /dev/null
if [ $? -ne 1 ]; then
log_error "some values are not set, check $ORIG_TEMPLATE_FILE"
echo ""
echo "missing keys in rendered template: $DEST_TEMPLATE_FILE"
echo " "
grep "<no value>" $DEST_TEMPLATE_FILE
exit 1
fi
}
###############################################################################
# MAIN
###############################################################################
set -o pipefail # enable strict command pipe error detection
check_requirements
create_dest_folder

download_kurtosis_artifacts

exit 1
fi
DEST=${TMP_CDK_FOLDER}/local_config

[ ! -d ${DEST} ] && mkdir -p ${DEST}
rm $DEST/*
kurtosis files download $KURTOSIS_ENCLAVE genesis $DEST
[ $? -ne 0 ] && echo "Error downloading genesis" && exit 1
export genesis_file=$DEST/genesis.json
export_values_of_genesis $genesis_file
kurtosis files download $KURTOSIS_ENCLAVE sequencer-keystore $DEST
[ $? -ne 0 ] && echo "Error downloading sequencer-keystore" && exit 1
export sequencer_keystore_file=$DEST/sequencer.keystore

l1_rpc_port=$(kurtosis port print $KURTOSIS_ENCLAVE el-1-geth-lighthouse rpc | cut -f 3 -d ":")
[ $? -ne 0 ] && echo "Error getting l1_rpc_port" && exit 1 || export l1_rpc_port && echo "l1_rpc_port=$l1_rpc_port"
l1_rpc_addr=$(kurtosis port print $KURTOSIS_ENCLAVE el-1-geth-lighthouse rpc)
[ $? -ne 0 ] && echo "Error getting l1_rpc_addr" && exit 1 || export l1_rpc_addr && echo "l1_rpc_addr=$l1_rpc_addr"
l2_rpc_addr=$(kurtosis port print $KURTOSIS_ENCLAVE cdk-erigon-node-001 http-rpc)
[ $? -ne 0 ] && echo "Error getting l2_rpc_addr" && exit 1 || export l2_rpc_addr && echo "l2_rpc_addr=$l2_rpc_addr"

zkevm_data_streamer_port=$(kurtosis port print $KURTOSIS_ENCLAVE cdk-erigon-sequencer-001 data-streamer | cut -f 3 -d ":")
[ $? -ne 0 ] && echo "Error getting zkevm_data_streamer_port" && exit 1 || export zkevm_data_streamer_port && echo "zkevm_data_streamer_port=$zkevm_data_streamer_port"

kurtosis files download $KURTOSIS_ENCLAVE cdk-node-config-artifact $DEST
export zkevm_l2_sequencer_address=$(cat $DEST/cdk-node-config.toml |grep L2Coinbase | cut -f 2 -d "="| tr -d '"' | tr -d ' ')
export zkevm_l2_keystore_password=$(cat $DEST/cdk-node-config.toml |grep -A1 L2Coinbase | tr ',' '\n' | grep Password | cut -f 2 -d '=' | tr -d '}' | tr -d '"' | tr -d ' ')
export l1_chain_id=$(cat $DEST/cdk-node-config.toml | grep L1ChainID | cut -f 2 -d '=' | head -n 1)
echo $l1_chain_id
export zkevm_is_validium=$(cat $DEST/cdk-node-config.toml | grep IsValidiumMode | cut -f 2 -d '=')
export zkevm_contract_versions=$(cat $DEST/cdk-node-config.toml | grep ContractVersions | cut -f 2 -d '=' | tr -d '"' | tr -d ' ')
if [ "$zkevm_is_validium" == "true" ]; then
echo "Validium mode detected... Retrieving the dac_port"
dac_port=$(kurtosis port print $KURTOSIS_ENCLAVE zkevm-dac-001 dac | cut -f 3 -d ":")
[ $? -ne 0 ] && echo "Error getting dac_port" && exit 1 || export dac_port && echo "dac_port=$dac_port"
fi

envsubst < test/config/test.kurtosis_template.toml > $DEST/test.kurtosis.toml
export_ports_from_kurtosis
export_values_of_cdk_node_config $DEST/cdk-node-config.toml
export_forced_values

ORIG_TEMPLATE_FILE=test/config/kurtosis-cdk-node-config.toml.template
DEST_TEMPLATE_FILE=$DEST/test.kurtosis.toml

# Generate config file
go run scripts/run_template.go $ORIG_TEMPLATE_FILE > $DEST_TEMPLATE_FILE
ok_or_fatal "Error generating template"

check_generated_config_file


echo " "
echo "file generated at:" $DEST/test.kurtosis.toml

echo "- to restart kurtosis:"
echo " kurtosis clean --all; kurtosis run --enclave cdk-v1 --args-file params.yml --image-download always ."
echo " "
Expand All @@ -102,7 +272,7 @@ cat << EOF
"cwd": "\${workspaceFolder}",
"args":[
"run",
"-cfg", "$DEST/test.kurtosis.toml",
"-cfg", "$DEST_TEMPLATE_FILE",
"-components", "sequence-sender,aggregator",
]
},
Expand Down
63 changes: 63 additions & 0 deletions scripts/run_template.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package main

import (
"fmt"
"log"
"os"
"regexp"
"strings"
"text/template"
)

func main() {
tmpl := template.New("t1")
content, err := readFile(os.Args[1])
if err != nil {
log.Fatalf("Error loading template: %v", err)
}
content = replaceDotsInTemplateVariables(content)
tmpl = template.Must(tmpl.Parse(content))

if err := tmpl.Execute(os.Stdout, environmentToMap()); err != nil {
log.Fatalf("Error executing template: %v", err)
}
}
func replaceDotsInTemplateVariables(template string) string {
re := regexp.MustCompile(`{{\s*\.([^{}]*)\s*}}`)
result := re.ReplaceAllStringFunc(template, func(match string) string {
match = strings.ReplaceAll(match[3:], ".", "_")
return "{{." + match
})
return result
}

func readFile(filename string) (string, error) {
content, err := os.ReadFile(filename)
if err != nil {
return "", err
}
return string(content), nil
}

func environmentToMap() map[string]any {
envVars := make(map[string]any)
for _, e := range os.Environ() {
pair := splitAtFirst(e, '=')
fmt.Printf("zzzz env=%s pair=%v\n", e, pair)
envVars[pair[0]] = pair[1]
}
envVars["aggregator_db"] = map[string]string{
"user": "user",
"name": "Name",
}
return envVars
}

func splitAtFirst(s string, sep rune) [2]string {
for i, c := range s {
if c == sep {
return [2]string{s[:i], s[i+1:]}
}
}
return [2]string{s, ""}
}
Loading

0 comments on commit 8167397

Please sign in to comment.