refactor: Migrate CDK aggregator pack2 assets to embed

Dockerfile

@@ -1,19 +1,23 @@
 # CONTAINER FOR BUILDING BINARY
 FROM golang:1.22.4 AS build
 
+WORKDIR $GOPATH/src/github.com/0xPolygon/cdk
+
 # INSTALL DEPENDENCIES
-RUN go install github.com/gobuffalo/packr/v2/[email protected]
-COPY go.mod go.sum /src/
-RUN cd /src && go mod download
+COPY go.mod go.sum ./
+RUN go mod download
 
 # BUILD BINARY
-COPY . /src
-RUN cd /src/aggregator/db && packr2
-RUN cd /src && make build
+COPY . .
+RUN make build
 
 # CONTAINER FOR RUNNING BINARY
 FROM alpine:3.18.4
-COPY --from=build /src/dist/cdk /app/cdk
+
+COPY --from=build /go/src/github.com/0xPolygon/cdk/dist/cdk /app/cdk
+
 RUN apk update && apk add postgresql15-client
+
 EXPOSE 8123
+
 CMD ["/bin/sh", "-c", "/app/cdk run"]

aggregator/db/db.go

@@ -5,119 +5,26 @@ import (
 	"fmt"
 
 	"github.com/0xPolygon/cdk/log"
-	"github.com/gobuffalo/packr/v2"
-	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/jackc/pgx/v4/stdlib"
-	migrate "github.com/rubenv/sql-migrate"
 )
 
-const (
-	// AggregatorMigrationName is the name of the migration used by packr to pack the migration file
-	AggregatorMigrationName = "zkevm-aggregator-db"
-)
-
-var packrMigrations = map[string]*packr.Box{
-	AggregatorMigrationName: packr.New(AggregatorMigrationName, "./migrations/aggregator"),
-}
-
 // NewSQLDB creates a new SQL DB
 func NewSQLDB(cfg Config) (*pgxpool.Pool, error) {
 	config, err := pgxpool.ParseConfig(fmt.Sprintf("postgres://%s:%s@%s:%s/%s?pool_max_conns=%d", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Name, cfg.MaxConns))
 	if err != nil {
 		log.Errorf("Unable to parse DB config: %v\n", err)
 		return nil, err
 	}
+
 	if cfg.EnableLog {
 		config.ConnConfig.Logger = logger{}
 	}
+
 	conn, err := pgxpool.ConnectConfig(context.Background(), config)
 	if err != nil {
 		log.Errorf("Unable to connect to database: %v\n", err)
 		return nil, err
 	}
-	return conn, nil
-}
-
-// RunMigrationsUp runs migrate-up for the given config.
-func RunMigrationsUp(cfg Config, name string) error {
-	log.Info("running migrations up")
-	return runMigrations(cfg, name, migrate.Up)
-}
-
-// CheckMigrations runs migrate-up for the given config.
-func CheckMigrations(cfg Config, name string) error {
-	return checkMigrations(cfg, name, migrate.Up)
-}
-
-// RunMigrationsDown runs migrate-down for the given config.
-func RunMigrationsDown(cfg Config, name string) error {
-	log.Info("running migrations down")
-	return runMigrations(cfg, name, migrate.Down)
-}
-
-// runMigrations will execute pending migrations if needed to keep
-// the database updated with the latest changes in either direction,
-// up or down.
-func runMigrations(cfg Config, packrName string, direction migrate.MigrationDirection) error {
-	c, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%s@%s:%s/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Name))
-	if err != nil {
-		return err
-	}
-	db := stdlib.OpenDB(*c)
-
-	box, ok := packrMigrations[packrName]
-	if !ok {
-		return fmt.Errorf("packr box not found with name: %v", packrName)
-	}
 
-	var migrations = &migrate.PackrMigrationSource{Box: box}
-	nMigrations, err := migrate.Exec(db, "postgres", migrations, direction)
-	if err != nil {
-		return err
-	}
-
-	log.Info("successfully ran ", nMigrations, " migrations")
-	return nil
-}
-
-func checkMigrations(cfg Config, packrName string, direction migrate.MigrationDirection) error {
-	c, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%s@%s:%s/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Name))
-	if err != nil {
-		return err
-	}
-	db := stdlib.OpenDB(*c)
-
-	box, ok := packrMigrations[packrName]
-	if !ok {
-		return fmt.Errorf("packr box not found with name: %v", packrName)
-	}
-
-	migrationSource := &migrate.PackrMigrationSource{Box: box}
-	migrations, err := migrationSource.FindMigrations()
-	if err != nil {
-		log.Errorf("error getting migrations from source: %v", err)
-		return err
-	}
-
-	var expected int
-	for _, migration := range migrations {
-		if len(migration.Up) != 0 {
-			expected++
-		}
-	}
-
-	var actual int
-	query := `SELECT COUNT(1) FROM public.gorp_migrations`
-	err = db.QueryRow(query).Scan(&actual)
-	if err != nil {
-		log.Error("error getting migrations count: ", err)
-		return err
-	}
-	if expected == actual {
-		log.Infof("Found %d migrations as expected", actual)
-	} else {
-		return fmt.Errorf("error the component needs to run %d migrations before starting. DB only contains %d migrations", expected, actual)
-	}
-	return nil
+	return conn, nil
 }

aggregator/db/migrations.go

@@ -0,0 +1,113 @@
+package db
+
+import (
+	"embed"
+	"fmt"
+
+	"github.com/0xPolygon/cdk/log"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/stdlib"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+const (
+	// AggregatorMigrationName is the name of the migration used to associate with the migrations dir
+	AggregatorMigrationName = "zkevm-aggregator-db"
+)
+
+var (
+	//go:embed migrations/aggregator/*.sql
+	embedAggregatorMigrations embed.FS
+
+	// embedMigrations is a map of migrations with the name
+	embedMigrations = map[string]embed.FS{}
+)
+
+func init() {
+	embedMigrations[AggregatorMigrationName] = embedAggregatorMigrations
+}
+
+// RunMigrationsUp runs migrate-up for the given config.
+func RunMigrationsUp(cfg Config, name string) error {
+	log.Info("running migrations up")
+	return runMigrations(cfg, name, migrate.Up)
+}
+
+// CheckMigrations runs migrate-up for the given config.
+func CheckMigrations(cfg Config, name string) error {
+	return checkMigrations(cfg, name)
+}
+
+// RunMigrationsDown runs migrate-down for the given config.
+func RunMigrationsDown(cfg Config, name string) error {
+	log.Info("running migrations down")
+	return runMigrations(cfg, name, migrate.Down)
+}
+
+// runMigrations will execute pending migrations if needed to keep
+// the database updated with the latest changes in either direction,
+// up or down.
+func runMigrations(cfg Config, name string, direction migrate.MigrationDirection) error {
+	c, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%s@%s:%s/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Name))
+	if err != nil {
+		return err
+	}
+
+	db := stdlib.OpenDB(*c)
+
+	embedMigration, ok := embedMigrations[name]
+	if !ok {
+		return fmt.Errorf("migration not found with name: %v", name)
+	}
+
+	var migrations = &migrate.EmbedFileSystemMigrationSource{FileSystem: embedMigration}
+	nMigrations, err := migrate.Exec(db, "postgres", migrations, direction)
+	if err != nil {
+		return err
+	}
+
+	log.Info("successfully ran ", nMigrations, " migrations")
+	return nil
+}
+
+func checkMigrations(cfg Config, name string) error {
+	c, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%s@%s:%s/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Name))
+	if err != nil {
+		return err
+	}
+
+	db := stdlib.OpenDB(*c)
+
+	embedMigration, ok := embedMigrations[name]
+	if !ok {
+		return fmt.Errorf("migration not found with name: %v", name)
+	}
+
+	migrationSource := &migrate.EmbedFileSystemMigrationSource{FileSystem: embedMigration}
+	migrations, err := migrationSource.FindMigrations()
+	if err != nil {
+		log.Errorf("error getting migrations from source: %v", err)
+		return err
+	}
+
+	var expected int
+	for _, migration := range migrations {
+		if len(migration.Up) != 0 {
+			expected++
+		}
+	}
+
+	var actual int
+	query := `SELECT COUNT(1) FROM public.gorp_migrations`
+	err = db.QueryRow(query).Scan(&actual)
+	if err != nil {
+		log.Error("error getting migrations count: ", err)
+		return err
+	}
+	if expected == actual {
+		log.Infof("Found %d migrations as expected", actual)
+	} else {
+		return fmt.Errorf("error the component needs to run %d migrations before starting. DB only contains %d migrations", expected, actual)
+	}
+	return nil
+}

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/0xPolygonHermez/zkevm-ethtx-manager v0.1.9
 	github.com/0xPolygonHermez/zkevm-synchronizer-l1 v0.6.1
 	github.com/ethereum/go-ethereum v1.14.5
-	github.com/gobuffalo/packr/v2 v2.8.3
 	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/iden3/go-iden3-crypto v0.0.16
 	github.com/invopop/jsonschema v0.12.0
@@ -69,6 +68,7 @@ require (
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/gobuffalo/logger v1.0.7 // indirect
 	github.com/gobuffalo/packd v1.0.2 // indirect
+	github.com/gobuffalo/packr/v2 v2.8.3 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect