-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9b3fa79
commit d9ce3ea
Showing
18 changed files
with
1,072 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| FROM ubuntu:bionic AS build | ||
|
|
||
| RUN apt-get update && apt-get install -y gnupg2 wget curl lsb-release git pkg-config \ | ||
| zip g++ zlib1g-dev unzip python libssl-dev default-jdk-headless libmariadbclient-dev | ||
|
|
||
| RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list | ||
| RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - | ||
| RUN apt-get update && apt-get install -y bazel | ||
|
|
||
| RUN git clone https://github.com/google/mysql-ripple.git && \ | ||
| cd mysql-ripple && \ | ||
| bazel build :all && \ | ||
| bazel test :all | ||
|
|
||
| FROM ubuntu:bionic | ||
|
|
||
| RUN apt-get update && apt-get install -y libssl-dev | ||
|
|
||
| COPY --from=build /mysql-ripple/bazel-bin/rippled /usr/local/bin/rippled | ||
|
|
||
| ENTRYPOINT ["rippled"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "flag" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| _ "github.com/go-sql-driver/mysql" | ||
|
|
||
| "github.com/github/gh-ost/go/localtests" | ||
| ) | ||
|
|
||
| var AppVersion string | ||
|
|
||
| func main() { | ||
| pwd, err := os.Getwd() | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
|
|
||
| var printVersion, testNoop bool | ||
| var localtestsDir, testName string | ||
| var cnf localtests.Config | ||
|
|
||
| flag.StringVar(&cnf.Host, "host", localtests.DefaultHost, "mysql host") | ||
| flag.Int64Var(&cnf.Port, "port", localtests.DefaultPort, "mysql port") | ||
| flag.StringVar(&cnf.Username, "username", localtests.DefaultUsername, "mysql username") | ||
| flag.StringVar(&cnf.Password, "password", localtests.DefaultPassword, "mysql password") | ||
| flag.StringVar(&localtestsDir, "tests-dir", filepath.Join(pwd, "localtests"), "path to localtests directory") | ||
| flag.StringVar(&testName, "test", "", "run a single test by name (default: run all tests)") | ||
| flag.BoolVar(&testNoop, "test-noop", false, "run a single noop migration, eg: --alter='ENGINE=InnoDB'") | ||
| flag.StringVar(&cnf.GhostBinary, "binary", "gh-ost", "path to gh-ost binary") | ||
| flag.StringVar(&cnf.MysqlBinary, "mysql-binary", "mysql", "path to mysql binary") | ||
| flag.BoolVar(&printVersion, "version", false, "print version and exit") | ||
| flag.Parse() | ||
|
|
||
| if printVersion { | ||
| fmt.Println(AppVersion) | ||
| os.Exit(0) | ||
| } | ||
|
|
||
| db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true", | ||
| cnf.Username, | ||
| cnf.Password, | ||
| cnf.Host, | ||
| cnf.Port, | ||
| )) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| defer db.Close() | ||
|
|
||
| if err = localtests.WaitForMySQLAvailable(db); err != nil { | ||
| log.Fatalf("Failed to setup database client: %+v", err) | ||
| } | ||
|
|
||
| var tests []localtests.Test | ||
| if testNoop { | ||
| tests = []localtests.Test{ | ||
| { | ||
| Name: "noop", | ||
| ExtraArgs: []string{ | ||
| `--alter='ENGINE=InnoDB'`, | ||
| }, | ||
| }, | ||
| } | ||
| } else { | ||
| tests, err = localtests.ReadTests(localtestsDir) | ||
| if err != nil { | ||
| log.Fatalf("Failed to read tests: %+v", err) | ||
| } | ||
| } | ||
|
|
||
| for _, test := range tests { | ||
| if testName != "" && test.Name != testName { | ||
| continue | ||
| } | ||
|
|
||
| log.Printf("Found test %q at %s/%s", test.Name, localtestsDir, test.Name) | ||
| if err = localtests.RunTest(db, cnf, test); err != nil { | ||
| log.Fatalf("Failed to run test %s: %+v", test.Name, err) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| package localtests | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "errors" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "log" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/google/shlex" | ||
| ) | ||
|
|
||
| const ( | ||
| PrimaryHost = "primary" | ||
| DefaultHost = "replica" | ||
| DefaultPort int64 = 3306 | ||
| DefaultUsername = "gh-ost" | ||
| DefaultPassword = "gh-ost" | ||
| testDatabase = "test" | ||
| testTable = "gh_ost_test" | ||
| testSocketFile = "/tmp/gh-ost.test.sock" | ||
| throttleFlagFile = "/tmp/gh-ost-test.ghost.throttle.flag" | ||
| throttleQuery = "select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc" | ||
| ) | ||
|
|
||
| type Config struct { | ||
| Host string | ||
| Port int64 | ||
| Username string | ||
| Password string | ||
| GhostBinary string | ||
| MysqlBinary string | ||
| } | ||
|
|
||
| type Test struct { | ||
| Name string | ||
| Path string | ||
| CreateSQLFile string | ||
| ExtraArgs []string | ||
| IgnoreVersions []string | ||
| } | ||
|
|
||
| func WaitForMySQLAvailable(db *sql.DB) error { | ||
| ticker := time.NewTicker(time.Second) | ||
| defer ticker.Stop() | ||
|
|
||
| for { | ||
| select { | ||
| case <-time.After(10 * time.Minute): | ||
| return errors.New("timed out waiting for mysql") | ||
| case <-ticker.C: | ||
| if err := db.Ping(); err != nil { | ||
| log.Println("Waiting for mysql to become available") | ||
| } else { | ||
| log.Println("MySQL is available") | ||
| return nil | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Prepare runs a 'mysql' client/shell command to populate the test schema. | ||
| // The create.sql file is read by golang and passed to 'mysql' over stdin. | ||
| func (test *Test) Prepare(config Config) error { | ||
| if test.CreateSQLFile == "" { | ||
| return nil | ||
| } | ||
|
|
||
| defaultsFile, err := writeMysqlClientDefaultsFile(config) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer os.Remove(defaultsFile) | ||
|
|
||
| flags := []string{ | ||
| fmt.Sprintf("--defaults-file=%s", defaultsFile), | ||
| fmt.Sprintf("--host=%s", PrimaryHost), // TODO: fix this | ||
| fmt.Sprintf("--port=%d", config.Port), | ||
| "--default-character-set=utf8mb4", | ||
| testDatabase, | ||
| } | ||
| log.Printf("[%s] running command: %s\n %s", test.Name, config.MysqlBinary, strings.Join(flags, "\n ")) | ||
|
|
||
| createSQL, err := os.Open(test.CreateSQLFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer createSQL.Close() | ||
| log.Printf("[%s] loaded sql from: %s", test.Name, test.CreateSQLFile) | ||
|
|
||
| cmd := exec.Command(config.MysqlBinary, flags...) | ||
| cmd.Stdin = createSQL | ||
| cmd.Stdout = os.Stdout | ||
| cmd.Stderr = os.Stderr | ||
| return cmd.Run() | ||
| } | ||
|
|
||
| func (test *Test) Migrate(db *sql.DB, config Config) error { | ||
| mysqlInfo, err := getMysqlHostInfo(db) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| log.Printf("[%s] detected MySQL %s host %s:%d", test.Name, mysqlInfo.Version, config.Host, config.Port) | ||
|
|
||
| flags := []string{ | ||
| fmt.Sprintf("--user=%s", config.Username), | ||
| fmt.Sprintf("--password=%s", config.Password), | ||
| fmt.Sprintf("--host=%s", config.Host), | ||
| fmt.Sprintf("--port=%d", config.Port), | ||
| fmt.Sprintf("--assume-master-host=primary:%d", mysqlInfo.Port), // TODO: fix this | ||
| fmt.Sprintf("--database=%s", testDatabase), | ||
| fmt.Sprintf("--table=%s", testTable), | ||
| "--assume-rbr", | ||
| "--chunk-size=10", | ||
| "--default-retries=3", | ||
| "--exact-rowcount", | ||
| "--initially-drop-old-table", | ||
| "--initially-drop-ghost-table", | ||
| "--initially-drop-socket-file", | ||
| fmt.Sprintf("--throttle-query=%s", throttleQuery), | ||
| fmt.Sprintf("--throttle-flag-file=%s", throttleFlagFile), | ||
| fmt.Sprintf("--serve-socket-file=%s", testSocketFile), | ||
| //"--test-on-replica", | ||
| "--allow-on-master", | ||
| "--debug", | ||
| "--execute", | ||
| "--stack", | ||
| "--verbose", | ||
| } | ||
| if len(test.ExtraArgs) > 0 { | ||
| flags = append(flags, test.ExtraArgs...) | ||
| } else { | ||
| flags = append(flags, `--alter='ENGINE=InnoDB'`) | ||
| } | ||
|
|
||
| log.Printf("[%s] running gh-ost command: %s\n %s", test.Name, config.GhostBinary, strings.Join(flags, "\n ")) | ||
| cmd := exec.Command(config.GhostBinary, flags...) | ||
| cmd.Stdout = os.Stdout | ||
| cmd.Stderr = os.Stderr | ||
| return cmd.Run() | ||
| } | ||
|
|
||
| func ReadTests(testsDir string) (tests []Test, err error) { | ||
| subdirs, err := ioutil.ReadDir(testsDir) | ||
| if err != nil { | ||
| return tests, err | ||
| } | ||
|
|
||
| for _, subdir := range subdirs { | ||
| test := Test{ | ||
| Name: subdir.Name(), | ||
| Path: filepath.Join(testsDir, subdir.Name()), | ||
| } | ||
|
|
||
| stat, err := os.Stat(test.Path) | ||
| if err != nil || !stat.IsDir() { | ||
| continue | ||
| } | ||
|
|
||
| test.CreateSQLFile = filepath.Join(test.Path, "create.sql") | ||
| if _, err = os.Stat(test.CreateSQLFile); err != nil { | ||
| log.Printf("Failed to find create.sql file %q: %+v", test.CreateSQLFile, err) | ||
| return tests, err | ||
| } | ||
|
|
||
| extraArgsFile := filepath.Join(test.Path, "extra_args") | ||
| if _, err = os.Stat(extraArgsFile); err == nil { | ||
| extraArgsStr, err := readTestFile(extraArgsFile) | ||
| if err != nil { | ||
| log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) | ||
| return tests, err | ||
| } | ||
| if test.ExtraArgs, err = shlex.Split(extraArgsStr); err != nil { | ||
| log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) | ||
| return tests, err | ||
| } | ||
| } | ||
|
|
||
| tests = append(tests, test) | ||
| } | ||
|
|
||
| return tests, err | ||
| } | ||
|
|
||
| func RunTest(db *sql.DB, config Config, test Test) error { | ||
| if err := test.Prepare(config); err != nil { | ||
| return err | ||
| } | ||
| log.Printf("[%s] prepared test", test.Name) | ||
|
|
||
| if err := test.Migrate(db, config); err != nil { | ||
| return err | ||
| } | ||
| log.Printf("[%s] migrated test", test.Name) | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.