feat: Integrate ethtxmanager with sql lite storage (#97)

* chore: fix comments

* feat: integrate the v0.2.0 eth tx manager spec

* fix: rename DBPath to StoragePath in the ethtxmanager config

* chore: rename according to golang guidelines

* chore: deprecate the PersistenceFilename config param

* test: unit test for obsoleted PersistenceFIlename configuration parameter

aggoracle/chaingersender/evm.go

@@ -9,7 +9,8 @@ import (
 	"github.com/0xPolygon/cdk-contracts-tooling/contracts/manual/pessimisticglobalexitroot"
 	cfgTypes "github.com/0xPolygon/cdk/config/types"
 	"github.com/0xPolygon/cdk/log"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
+	ethtxtypes "github.com/0xPolygon/zkevm-ethtx-manager/types"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
@@ -26,9 +27,9 @@ type EthClienter interface {
 type EthTxManager interface {
 	Remove(ctx context.Context, id common.Hash) error
 	ResultsByStatus(ctx context.Context,
-		statuses []ethtxmanager.MonitoredTxStatus,
-	) ([]ethtxmanager.MonitoredTxResult, error)
-	Result(ctx context.Context, id common.Hash) (ethtxmanager.MonitoredTxResult, error)
+		statuses []ethtxtypes.MonitoredTxStatus,
+	) ([]ethtxtypes.MonitoredTxResult, error)
+	Result(ctx context.Context, id common.Hash) (ethtxtypes.MonitoredTxResult, error)
 	Add(ctx context.Context,
 		to *common.Address,
 		value *big.Int,
@@ -114,14 +115,14 @@ func (c *EVMChainGERSender) UpdateGERWaitUntilMined(ctx context.Context, ger com
 			c.logger.Error("error calling ethTxMan.Result: ", err)
 		}
 		switch res.Status {
-		case ethtxmanager.MonitoredTxStatusCreated,
-			ethtxmanager.MonitoredTxStatusSent:
+		case ethtxtypes.MonitoredTxStatusCreated,
+			ethtxtypes.MonitoredTxStatusSent:
 			continue
-		case ethtxmanager.MonitoredTxStatusFailed:
+		case ethtxtypes.MonitoredTxStatusFailed:
 			return fmt.Errorf("tx %s failed", res.ID)
-		case ethtxmanager.MonitoredTxStatusMined,
-			ethtxmanager.MonitoredTxStatusSafe,
-			ethtxmanager.MonitoredTxStatusFinalized:
+		case ethtxtypes.MonitoredTxStatusMined,
+			ethtxtypes.MonitoredTxStatusSafe,
+			ethtxtypes.MonitoredTxStatusFinalized:
 			return nil
 		default:
 			c.logger.Error("unexpected tx status: ", res.Status)

aggregator/aggregator.go

@@ -24,10 +24,11 @@ import (
 	"github.com/0xPolygon/cdk/log"
 	"github.com/0xPolygon/cdk/state"
 	"github.com/0xPolygon/cdk/state/datastream"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
+	ethtxlog "github.com/0xPolygon/zkevm-ethtx-manager/log"
+	ethtxtypes "github.com/0xPolygon/zkevm-ethtx-manager/types"
 	"github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer"
 	streamlog "github.com/0xPolygonHermez/zkevm-data-streamer/log"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
-	ethtxlog "github.com/0xPolygonHermez/zkevm-ethtx-manager/log"
 	synclog "github.com/0xPolygonHermez/zkevm-synchronizer-l1/log"
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/state/entities"
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/synchronizer"
@@ -989,7 +990,7 @@ func (a *Aggregator) settleDirect(
 	}
 
 	// process monitored batch verifications before starting a next cycle
-	a.ethTxManager.ProcessPendingMonitoredTxs(ctx, func(result ethtxmanager.MonitoredTxResult) {
+	a.ethTxManager.ProcessPendingMonitoredTxs(ctx, func(result ethtxtypes.MonitoredTxResult) {
 		a.handleMonitoredTxResult(result)
 	})
 
@@ -1928,9 +1929,9 @@ func (hc *healthChecker) Watch(req *grpchealth.HealthCheckRequest, server grpche
 	})
 }
 
-func (a *Aggregator) handleMonitoredTxResult(result ethtxmanager.MonitoredTxResult) {
+func (a *Aggregator) handleMonitoredTxResult(result ethtxtypes.MonitoredTxResult) {
 	mTxResultLogger := ethtxmanager.CreateMonitoredTxResultLogger(result)
-	if result.Status == ethtxmanager.MonitoredTxStatusFailed {
+	if result.Status == ethtxtypes.MonitoredTxStatusFailed {
 		mTxResultLogger.Fatal("failed to send batch verification, TODO: review this fatal and define what to do in this case")
 	}
 

aggregator/config.go

@@ -10,7 +10,7 @@ import (
 	"github.com/0xPolygon/cdk/aggregator/db"
 	"github.com/0xPolygon/cdk/config/types"
 	"github.com/0xPolygon/cdk/log"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
 	syncronizerConfig "github.com/0xPolygonHermez/zkevm-synchronizer-l1/config"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 )

claimsponsor/evmclaimsponsor.go

@@ -9,7 +9,8 @@ import (
 	"github.com/0xPolygon/cdk-contracts-tooling/contracts/etrog/polygonzkevmbridgev2"
 	configTypes "github.com/0xPolygon/cdk/config/types"
 	"github.com/0xPolygon/cdk/log"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
+	ethtxtypes "github.com/0xPolygon/zkevm-ethtx-manager/types"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -33,9 +34,8 @@ type EthClienter interface {
 
 type EthTxManager interface {
 	Remove(ctx context.Context, id common.Hash) error
-	ResultsByStatus(ctx context.Context, statuses []ethtxmanager.MonitoredTxStatus,
-	) ([]ethtxmanager.MonitoredTxResult, error)
-	Result(ctx context.Context, id common.Hash) (ethtxmanager.MonitoredTxResult, error)
+	ResultsByStatus(ctx context.Context, statuses []ethtxtypes.MonitoredTxStatus) ([]ethtxtypes.MonitoredTxResult, error)
+	Result(ctx context.Context, id common.Hash) (ethtxtypes.MonitoredTxResult, error)
 	Add(ctx context.Context, to *common.Address, value *big.Int, data []byte,
 		gasOffset uint64, sidecar *types.BlobTxSidecar) (common.Hash, error)
 }
@@ -166,14 +166,14 @@ func (c *EVMClaimSponsor) claimStatus(ctx context.Context, id string) (ClaimStat
 		return "", err
 	}
 	switch res.Status {
-	case ethtxmanager.MonitoredTxStatusCreated,
-		ethtxmanager.MonitoredTxStatusSent:
+	case ethtxtypes.MonitoredTxStatusCreated,
+		ethtxtypes.MonitoredTxStatusSent:
 		return WIPStatus, nil
-	case ethtxmanager.MonitoredTxStatusFailed:
+	case ethtxtypes.MonitoredTxStatusFailed:
 		return FailedClaimStatus, nil
-	case ethtxmanager.MonitoredTxStatusMined,
-		ethtxmanager.MonitoredTxStatusSafe,
-		ethtxmanager.MonitoredTxStatusFinalized:
+	case ethtxtypes.MonitoredTxStatusMined,
+		ethtxtypes.MonitoredTxStatusSafe,
+		ethtxtypes.MonitoredTxStatusFinalized:
 		return SuccessClaimStatus, nil
 	default:
 		return "", fmt.Errorf("unexpected tx status: %v", res.Status)

cmd/run.go

@@ -35,10 +35,10 @@ import (
 	"github.com/0xPolygon/cdk/state"
 	"github.com/0xPolygon/cdk/state/pgstatestorage"
 	"github.com/0xPolygon/cdk/translator"
-	ethtxman "github.com/0xPolygonHermez/zkevm-ethtx-manager/etherman"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/etherman/etherscan"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
-	ethtxlog "github.com/0xPolygonHermez/zkevm-ethtx-manager/log"
+	ethtxman "github.com/0xPolygon/zkevm-ethtx-manager/etherman"
+	"github.com/0xPolygon/zkevm-ethtx-manager/etherman/etherscan"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
+	ethtxlog "github.com/0xPolygon/zkevm-ethtx-manager/log"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/urfave/cli/v2"

config/config.go

@@ -18,7 +18,7 @@ import (
 	"github.com/0xPolygon/cdk/log"
 	"github.com/0xPolygon/cdk/reorgdetector"
 	"github.com/0xPolygon/cdk/sequencesender"
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/ethtxmanager"
+	"github.com/0xPolygon/zkevm-ethtx-manager/ethtxmanager"
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/urfave/cli/v2"
@@ -51,7 +51,10 @@ const (
 	// FlagMaxAmount is the flag to avoid to use the flag FlagAmount
 	FlagMaxAmount = "max-amount"
 
-	deprecatedFieldSyncDB = "Aggregator.Synchronizer.DB is deprecated use Aggregator.Synchronizer.SQLDB instead"
+	deprecatedFieldSyncDB = "Aggregator.Synchronizer.DB is deprecated. Use Aggregator.Synchronizer.SQLDB instead."
+
+	deprecatedFieldPersistenceFilename = "EthTxManager.PersistenceFilename is deprecated." +
+		" Use EthTxManager.StoragePath instead."
 )
 
 type ForbiddenField struct {
@@ -65,6 +68,14 @@ var (
 			FieldName: "aggregator.synchronizer.db.",
 			Reason:    deprecatedFieldSyncDB,
 		},
+		{
+			FieldName: "sequencesender.ethtxmanager.persistencefilename",
+			Reason:    deprecatedFieldPersistenceFilename,
+		},
+		{
+			FieldName: "aggregator.ethtxmanager.persistencefilename",
+			Reason:    deprecatedFieldPersistenceFilename,
+		},
 	}
 )
 

config/config_test.go

@@ -34,18 +34,33 @@ func TestLoadConfigWithUnexpectedFields(t *testing.T) {
 	require.NotNil(t, cfg)
 }
 
-const configWithForbiddenFields = `
-[aggregator.synchronizer.db]
-name = "value"
-`
-
 func TestLoadConfigWithForbiddenFields(t *testing.T) {
-	tmpFile, err := os.CreateTemp("", "ut_config")
-	require.NoError(t, err)
-	defer os.Remove(tmpFile.Name())
-	_, err = tmpFile.Write([]byte(configWithForbiddenFields))
-	require.NoError(t, err)
-	cfg, err := LoadFile(tmpFile.Name())
-	require.NoError(t, err)
-	require.NotNil(t, cfg)
+	cases := []struct {
+		name  string
+		input string
+	}{
+		{
+			name: "[Aggregator.Synchronizer] DB",
+			input: `[aggregator.synchronizer.db]
+						name = "value"`,
+		},
+		{
+			name: "[SequenceSender.EthTxManager] PersistenceFilename",
+			input: `[SequenceSender.EthTxManager]
+						PersistenceFilename = "foo.json"`,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			tmpFile, err := os.CreateTemp("", "ut_config")
+			require.NoError(t, err)
+			defer os.Remove(tmpFile.Name())
+			_, err = tmpFile.Write([]byte(c.input))
+			require.NoError(t, err)
+			cfg, err := LoadFile(tmpFile.Name())
+			require.NoError(t, err)
+			require.NotNil(t, cfg)
+		})
+	}
 }

config/default.go

@@ -53,7 +53,7 @@ GetBatchWaitInterval = "10s"
 		ForcedGas = 0
 		GasPriceMarginFactor = 1
 		MaxGasPriceLimit = 0
-		PersistenceFilename = "ethtxmanager.json"
+		StoragePath = "ethtxmanager.db"
 		ReadPendingL1Txs = false
 		SafeStatusL1NumberOfBlocks = 0
 		FinalizedStatusL1NumberOfBlocks = 0
@@ -109,7 +109,7 @@ SequencerPrivateKey = {}
 		ForcedGas = 0
 		GasPriceMarginFactor = 1
 		MaxGasPriceLimit = 0
-		PersistenceFilename = ""
+		StoragePath = ""
 		ReadPendingL1Txs = false
 		SafeStatusL1NumberOfBlocks = 0
 		FinalizedStatusL1NumberOfBlocks = 0
@@ -184,7 +184,7 @@ WaitPeriodNextGER="100ms"
 				ForcedGas = 0
 				GasPriceMarginFactor = 1
 				MaxGasPriceLimit = 0
-				PersistenceFilename = "/tmp/ethtxmanager-sequencesender.json"
+				StoragePath = "/tmp/ethtxmanager-sequencesender.db"
 				ReadPendingL1Txs = false
 				SafeStatusL1NumberOfBlocks = 5
 				FinalizedStatusL1NumberOfBlocks = 10
@@ -223,7 +223,7 @@ GasOffset = 0
 		ForcedGas = 0
 		GasPriceMarginFactor = 1
 		MaxGasPriceLimit = 0
-		PersistenceFilename = "/tmp/ethtxmanager-claimsopnsor.json"
+		StoragePath = "/tmp/ethtxmanager-claimsponsor.db"
 		ReadPendingL1Txs = false
 		SafeStatusL1NumberOfBlocks = 5
 		FinalizedStatusL1NumberOfBlocks = 10

config/example-config.toml

@@ -31,7 +31,7 @@ MaxPendingTx = 1
 		ForcedGas = 0
 		GasPriceMarginFactor = 1
 		MaxGasPriceLimit = 0
-		PersistenceFilename = "ethtxmanager.json"
+		StoragePath = "ethtxmanager.db"
 		ReadPendingL1Txs = false
 		SafeStatusL1NumberOfBlocks = 0
 		FinalizedStatusL1NumberOfBlocks = 0
@@ -84,7 +84,7 @@ SequencerPrivateKey = {}
 		ForcedGas = 0
 		GasPriceMarginFactor = 1
 		MaxGasPriceLimit = 0
-		PersistenceFilename = ""
+		StoragePath = ""
 		ReadPendingL1Txs = false
 		SafeStatusL1NumberOfBlocks = 0
 		FinalizedStatusL1NumberOfBlocks = 0

db/meddler.go

@@ -38,7 +38,7 @@ func SliceToSlicePtrs(slice interface{}) interface{} {
 	v := reflect.ValueOf(slice)
 	vLen := v.Len()
 	typ := v.Type().Elem()
-	res := reflect.MakeSlice(reflect.SliceOf(reflect.PtrTo(typ)), vLen, vLen)
+	res := reflect.MakeSlice(reflect.SliceOf(reflect.PointerTo(typ)), vLen, vLen)
 	for i := 0; i < vLen; i++ {
 		res.Index(i).Set(v.Index(i).Addr())
 	}
@@ -57,7 +57,7 @@ func SlicePtrsToSlice(slice interface{}) interface{} {
 	return res.Interface()
 }
 
-// BigIntMeddler encodes or decodes the field value to or from JSON
+// BigIntMeddler encodes or decodes the field value to or from string
 type BigIntMeddler struct{}
 
 // PreRead is called before a Scan operation for fields that have the BigIntMeddler
@@ -97,16 +97,16 @@ func (b BigIntMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, er
 	return field.String(), nil
 }
 
-// MerkleProofMeddler encodes or decodes the field value to or from JSON
+// MerkleProofMeddler encodes or decodes the field value to or from string
 type MerkleProofMeddler struct{}
 
-// PreRead is called before a Scan operation for fields that have the ProofMeddler
+// PreRead is called before a Scan operation for fields that have the MerkleProofMeddler
 func (b MerkleProofMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) {
 	// give a pointer to a byte buffer to grab the raw data
 	return new(string), nil
 }
 
-// PostRead is called after a Scan operation for fields that have the ProofMeddler
+// PostRead is called after a Scan operation for fields that have the MerkleProofMeddler
 func (b MerkleProofMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	ptr, ok := scanTarget.(*string)
 	if !ok {
@@ -129,7 +129,7 @@ func (b MerkleProofMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	return nil
 }
 
-// PreWrite is called before an Insert or Update operation for fields that have the ProofMeddler
+// PreWrite is called before an Insert or Update operation for fields that have the MerkleProofMeddler
 func (b MerkleProofMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
 	field, ok := fieldPtr.(tree.Proof)
 	if !ok {
@@ -143,16 +143,16 @@ func (b MerkleProofMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{
 	return s, nil
 }
 
-// HashMeddler encodes or decodes the field value to or from JSON
+// HashMeddler encodes or decodes the field value to or from string
 type HashMeddler struct{}
 
-// PreRead is called before a Scan operation for fields that have the ProofMeddler
+// PreRead is called before a Scan operation for fields that have the HashMeddler
 func (b HashMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) {
 	// give a pointer to a byte buffer to grab the raw data
 	return new(string), nil
 }
 
-// PostRead is called after a Scan operation for fields that have the ProofMeddler
+// PostRead is called after a Scan operation for fields that have the HashMeddler
 func (b HashMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	ptr, ok := scanTarget.(*string)
 	if !ok {
@@ -169,7 +169,7 @@ func (b HashMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	return nil
 }
 
-// PreWrite is called before an Insert or Update operation for fields that have the ProofMeddler
+// PreWrite is called before an Insert or Update operation for fields that have the HashMeddler
 func (b HashMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
 	field, ok := fieldPtr.(common.Hash)
 	if !ok {
@@ -178,16 +178,16 @@ func (b HashMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err
 	return field.Hex(), nil
 }
 
-// AddressMeddler encodes or decodes the field value to or from JSON
+// AddressMeddler encodes or decodes the field value to or from string
 type AddressMeddler struct{}
 
-// PreRead is called before a Scan operation for fields that have the ProofMeddler
+// PreRead is called before a Scan operation for fields that have the AddressMeddler
 func (b AddressMeddler) PreRead(fieldAddr interface{}) (scanTarget interface{}, err error) {
 	// give a pointer to a byte buffer to grab the raw data
 	return new(string), nil
 }
 
-// PostRead is called after a Scan operation for fields that have the ProofMeddler
+// PostRead is called after a Scan operation for fields that have the AddressMeddler
 func (b AddressMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	ptr, ok := scanTarget.(*string)
 	if !ok {
@@ -204,7 +204,7 @@ func (b AddressMeddler) PostRead(fieldPtr, scanTarget interface{}) error {
 	return nil
 }
 
-// PreWrite is called before an Insert or Update operation for fields that have the ProofMeddler
+// PreWrite is called before an Insert or Update operation for fields that have the AddressMeddler
 func (b AddressMeddler) PreWrite(fieldPtr interface{}) (saveValue interface{}, err error) {
 	field, ok := fieldPtr.(common.Address)
 	if !ok {

etherman/config/config.go

@@ -1,7 +1,7 @@
 package config
 
 import (
-	"github.com/0xPolygonHermez/zkevm-ethtx-manager/etherman"
+	"github.com/0xPolygon/zkevm-ethtx-manager/etherman"
 	"github.com/ethereum/go-ethereum/common"
 )