Merge pull request ethereum-optimism#3767 from ethereum-optimism/feat…

…/db-migration

op-chain-ops: implement migration script

op-chain-ops/cmd/migrate/main.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"context"
+	"math/big"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum-optimism/optimism/l2geth/core/rawdb"
+	"github.com/ethereum-optimism/optimism/l2geth/core/state"
+	"github.com/ethereum-optimism/optimism/l2geth/log"
+	"github.com/ethereum-optimism/optimism/op-chain-ops/genesis"
+
+	op_state "github.com/ethereum-optimism/optimism/op-chain-ops/state"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethclient"
+
+	"github.com/mattn/go-isatty"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	log.Root().SetHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(isatty.IsTerminal(os.Stderr.Fd()))))
+
+	app := &cli.App{
+		Name:  "migrate",
+		Usage: "Migrate a legacy database",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "l1-rpc-url",
+				Value: "http://127.0.0.1:8545",
+				Usage: "RPC URL for an L1 Node",
+			},
+			&cli.Uint64Flag{
+				Name:  "starting-l1-block-number",
+				Usage: "L1 block number to build the L2 genesis from",
+			},
+			&cli.StringFlag{
+				Name:  "ovm-addresses",
+				Usage: "Path to ovm-addresses.json",
+			},
+			&cli.StringFlag{
+				Name:  "evm-addresses",
+				Usage: "Path to evm-addresses.json",
+			},
+			&cli.StringFlag{
+				Name:  "ovm-allowances",
+				Usage: "Path to ovm-allowances.json",
+			},
+			&cli.StringFlag{
+				Name:  "ovm-messages",
+				Usage: "Path to ovm-messages.json",
+			},
+			&cli.StringFlag{
+				Name:  "evm-messages",
+				Usage: "Path to evm-messages.json",
+			},
+			&cli.StringFlag{
+				Name:  "db-path",
+				Usage: "Path to database",
+			},
+			cli.StringFlag{
+				Name:  "deploy-config",
+				Usage: "Path to hardhat deploy config file",
+			},
+			cli.BoolFlag{
+				Name:  "dry-run",
+				Usage: "Dry run the upgrade by not committing the database",
+			},
+		},
+		Action: func(ctx *cli.Context) error {
+			deployConfig := ctx.String("deploy-config")
+			config, err := genesis.NewDeployConfig(deployConfig)
+			if err != nil {
+				return err
+			}
+
+			ovmAddresses, err := genesis.NewAddresses(ctx.String("ovm-addresses"))
+			if err != nil {
+				return err
+			}
+			evmAddresess, err := genesis.NewAddresses(ctx.String("evm-addresses"))
+			if err != nil {
+				return err
+			}
+			ovmAllowances, err := genesis.NewAllowances(ctx.String("ovm-allowances"))
+			if err != nil {
+				return err
+			}
+			ovmMessages, err := genesis.NewSentMessage(ctx.String("ovm-messages"))
+			if err != nil {
+				return err
+			}
+			evmMessages, err := genesis.NewSentMessage(ctx.String("evm-messages"))
+			if err != nil {
+				return err
+			}
+
+			migrationData := genesis.MigrationData{
+				OvmAddresses:  ovmAddresses,
+				EvmAddresses:  evmAddresess,
+				OvmAllowances: ovmAllowances,
+				OvmMessages:   ovmMessages,
+				EvmMessages:   evmMessages,
+			}
+
+			l1RpcURL := ctx.String("l1-rpc-url")
+			l1Client, err := ethclient.Dial(l1RpcURL)
+			if err != nil {
+				return err
+			}
+			var blockNumber *big.Int
+			bnum := ctx.Uint64("starting-l1-block-number")
+			if bnum != 0 {
+				blockNumber = new(big.Int).SetUint64(bnum)
+			}
+
+			block, err := l1Client.BlockByNumber(context.Background(), blockNumber)
+			if err != nil {
+				return err
+			}
+
+			chaindataPath := filepath.Join(ctx.String("db-path"), "geth", "chaindata")
+			ldb, err := rawdb.NewLevelDBDatabase(chaindataPath, 1024, 64, "")
+			if err != nil {
+				return err
+			}
+
+			hash := rawdb.ReadHeadHeaderHash(ldb)
+			if err != nil {
+				return err
+			}
+			num := rawdb.ReadHeaderNumber(ldb, hash)
+			header := rawdb.ReadHeader(ldb, hash, *num)
+
+			sdb, err := state.New(header.Root, state.NewDatabase(ldb))
+			if err != nil {
+				return err
+			}
+			wrappedDB, err := op_state.NewWrappedStateDB(nil, sdb)
+			if err != nil {
+				return err
+			}
+
+			l2Addrs := genesis.L2Addresses{
+				ProxyAdminOwner: config.ProxyAdminOwner,
+				// TODO: these values are not in the config
+				L1StandardBridgeProxy:       common.Address{},
+				L1CrossDomainMessengerProxy: common.Address{},
+				L1ERC721BridgeProxy:         common.Address{},
+			}
+
+			if err := genesis.MigrateDB(wrappedDB, config, block, &l2Addrs, &migrationData); err != nil {
+				return err
+			}
+
+			if ctx.Bool("dry-run") {
+				log.Info("Dry run complete")
+				return nil
+			}
+
+			root, err := sdb.Commit(true)
+			if err != nil {
+				return err
+			}
+			log.Info("Migration complete", "root", root)
+
+			return nil
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Crit("error in migration", "err", err)
+	}
+}

op-chain-ops/crossdomain/migrate.go

@@ -18,7 +18,7 @@ var (
 )
 
 // MigrateWithdrawals will migrate a list of pending withdrawals given a StateDB.
-func MigrateWithdrawals(withdrawals []*PendingWithdrawal, db vm.StateDB, l1CrossDomainMessenger, l1StandardBridge *common.Address) error {
+func MigrateWithdrawals(withdrawals []*LegacyWithdrawal, db vm.StateDB, l1CrossDomainMessenger, l1StandardBridge *common.Address) error {
 	for _, legacy := range withdrawals {
 		legacySlot, err := legacy.StorageSlot()
 		if err != nil {
@@ -30,7 +30,7 @@ func MigrateWithdrawals(withdrawals []*PendingWithdrawal, db vm.StateDB, l1Cross
 			return fmt.Errorf("%w: %s", errLegacyStorageSlotNotFound, legacyValue)
 		}
 
-		withdrawal, err := MigrateWithdrawal(&legacy.LegacyWithdrawal, l1CrossDomainMessenger, l1StandardBridge)
+		withdrawal, err := MigrateWithdrawal(legacy, l1CrossDomainMessenger, l1StandardBridge)
 		if err != nil {
 			return err
 		}

op-chain-ops/genesis/db_migration.go

@@ -0,0 +1,53 @@
+package genesis
+
+import (
+	"github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+)
+
+// MigrateDB will migrate an old l2geth database to the new bedrock style system
+func MigrateDB(db vm.StateDB, config *DeployConfig, l1Block *types.Block, l2Addrs *L2Addresses, migrationData *MigrationData) error {
+	if err := SetL2Proxies(db); err != nil {
+		return err
+	}
+
+	storage, err := NewL2StorageConfig(config, l1Block, l2Addrs)
+	if err != nil {
+		return err
+	}
+
+	immutable, err := NewL2ImmutableConfig(config, l1Block, l2Addrs)
+	if err != nil {
+		return err
+	}
+
+	if err := SetImplementations(db, storage, immutable); err != nil {
+		return err
+	}
+
+	// Convert all of the messages into legacy withdrawals
+	messages := make([]*crossdomain.LegacyWithdrawal, 0)
+	for _, msg := range migrationData.OvmMessages {
+		wd, err := msg.ToLegacyWithdrawal()
+		if err != nil {
+			return err
+		}
+		messages = append(messages, wd)
+	}
+	for _, msg := range migrationData.EvmMessages {
+		wd, err := msg.ToLegacyWithdrawal()
+		if err != nil {
+			return err
+		}
+		messages = append(messages, wd)
+	}
+
+	if err := crossdomain.MigrateWithdrawals(messages, db, &l2Addrs.L1CrossDomainMessengerProxy, &l2Addrs.L1StandardBridgeProxy); err != nil {
+		return err
+	}
+
+	// TODO: use migration data to double check things
+
+	return nil
+}

op-chain-ops/genesis/migration.go

@@ -0,0 +1,118 @@
+package genesis
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/ethereum-optimism/optimism/op-chain-ops/crossdomain"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// SentMessageJSON represents an entry in the JSON file that is created by
+// the `migration-data` package. Each entry represents a call to the
+// `LegacyMessagePasser`. The `who` should always be the
+// `L2CrossDomainMessenger` and the `msg` should be an abi encoded
+// `relayMessage(address,address,bytes,uint256)`
+type SentMessage struct {
+	Who common.Address `json:"who"`
+	Msg hexutil.Bytes  `json:"msg"`
+}
+
+// NewSentMessageJSON will read a JSON file from disk given a path to the JSON
+// file. The JSON file this function reads from disk is an output from the
+// `migration-data` package.
+func NewSentMessage(path string) ([]*SentMessage, error) {
+	file, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot find sent message json at %s: %w", path, err)
+	}
+
+	var j []*SentMessage
+	if err := json.Unmarshal(file, &j); err != nil {
+		return nil, err
+	}
+
+	return j, nil
+}
+
+// ToLegacyWithdrawal will convert a SentMessageJSON to a LegacyWithdrawal
+// struct. This is useful because the LegacyWithdrawal struct has helper
+// functions on it that can compute the withdrawal hash and the storage slot.
+func (s *SentMessage) ToLegacyWithdrawal() (*crossdomain.LegacyWithdrawal, error) {
+	data := make([]byte, 0, len(s.Who)+len(s.Msg))
+	copy(data, s.Msg)
+	copy(data[len(s.Msg):], s.Who[:])
+
+	var w crossdomain.LegacyWithdrawal
+	if err := w.Decode(data); err != nil {
+		return nil, err
+	}
+	return &w, nil
+}
+
+// OVMETHAddresses represents a list of addresses that interacted with
+// the ERC20 representation of ether in the pre-bedrock system.
+type OVMETHAddresses map[common.Address]bool
+
+// NewAddresses will read an addresses.json file from the filesystem.
+func NewAddresses(path string) (OVMETHAddresses, error) {
+	file, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot find addresses json at %s: %w", path, err)
+	}
+
+	var addresses []common.Address
+	if err := json.Unmarshal(file, &addresses); err != nil {
+		return nil, err
+	}
+
+	ovmeth := make(OVMETHAddresses)
+	for _, addr := range addresses {
+		ovmeth[addr] = true
+	}
+
+	return ovmeth, nil
+}
+
+// Allowance represents the allowances that were set in the
+// legacy ERC20 representation of ether
+type Allowance struct {
+	From common.Address `json:"fr"`
+	To   common.Address `json:"to"`
+}
+
+// NewAllowances will read the ovm-allowances.json from the file system.
+func NewAllowances(path string) ([]*Allowance, error) {
+	file, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot find allowances json at %s: %w", path, err)
+	}
+
+	var allowances []*Allowance
+	if err := json.Unmarshal(file, &allowances); err != nil {
+		return nil, err
+	}
+
+	return allowances, nil
+}
+
+// MigrationData represents all of the data required to do a migration
+type MigrationData struct {
+	// OvmAddresses represents the set of addresses that interacted with the
+	// LegacyERC20ETH contract before the evm equivalence upgrade
+	OvmAddresses OVMETHAddresses
+	// EvmAddresses represents the set of addresses that interacted with the
+	// LegacyERC20ETH contract after the evm equivalence upgrade
+	EvmAddresses OVMETHAddresses
+	// OvmAllowances represents the set of allowances in the LegacyERC20ETH from
+	// before the evm equivalence upgrade
+	OvmAllowances []*Allowance
+	// OvmMessages represents the set of withdrawals through the
+	// L2CrossDomainMessenger from before the evm equivalence upgrade
+	OvmMessages []*SentMessage
+	// OvmMessages represents the set of withdrawals through the
+	// L2CrossDomainMessenger from after the evm equivalence upgrade
+	EvmMessages []*SentMessage
+}

op-chain-ops/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/holiman/uint256 v1.2.0
 	github.com/mattn/go-isatty v0.0.14
 	github.com/stretchr/testify v1.8.0
+	github.com/urfave/cli v1.22.1
 	github.com/urfave/cli/v2 v2.10.2
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e
 )

op-chain-ops/go.sum

@@ -613,6 +613,7 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s
 github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
 github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y=