diff --git a/Dockerfile.ripple b/Dockerfile.ripple new file mode 100644 index 000000000..3c6561c1c --- /dev/null +++ b/Dockerfile.ripple @@ -0,0 +1,21 @@ +FROM ubuntu:bionic AS build + +RUN apt-get update && apt-get install -y gnupg2 wget curl lsb-release git pkg-config \ + zip g++ zlib1g-dev unzip python libssl-dev default-jdk-headless libmariadbclient-dev + +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get update && apt-get install -y bazel + +RUN git clone https://github.com/google/mysql-ripple.git && \ + cd mysql-ripple && \ + bazel build :all && \ + bazel test :all + +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y libssl-dev + +COPY --from=build /mysql-ripple/bazel-bin/rippled /usr/local/bin/rippled + +ENTRYPOINT ["rippled"] diff --git a/go.mod b/go.mod index 885d97eeb..d25669c57 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/go-ini/ini v1.62.0 github.com/go-mysql-org/go-mysql v1.3.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 github.com/satori/go.uuid v1.2.0 - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 golang.org/x/text v0.3.6 ) @@ -21,7 +22,6 @@ require ( github.com/smartystreets/goconvey v1.6.4 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index 6b44084a0..a2cf26214 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= @@ -81,8 +83,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -96,14 +96,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/go/cmd/gh-ost-localtests/main.go b/go/cmd/gh-ost-localtests/main.go new file mode 100644 index 000000000..9e684392a --- /dev/null +++ b/go/cmd/gh-ost-localtests/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "log" + "os" + "path/filepath" + + _ "github.com/go-sql-driver/mysql" + + "github.com/github/gh-ost/go/localtests" +) + +var AppVersion string + +func main() { + pwd, err := os.Getwd() + if err != nil { + panic(err) + } + + var printVersion, testNoop bool + var localtestsDir, testName string + var cnf localtests.Config + + flag.StringVar(&cnf.Host, "host", localtests.DefaultHost, "mysql host") + flag.Int64Var(&cnf.Port, "port", localtests.DefaultPort, "mysql port") + flag.StringVar(&cnf.Username, "username", localtests.DefaultUsername, "mysql username") + flag.StringVar(&cnf.Password, "password", localtests.DefaultPassword, "mysql password") + flag.StringVar(&localtestsDir, "tests-dir", filepath.Join(pwd, "localtests"), "path to localtests directory") + flag.StringVar(&testName, "test", "", "run a single test by name (default: run all tests)") + flag.BoolVar(&testNoop, "test-noop", false, "run a single noop migration, eg: --alter='ENGINE=InnoDB'") + flag.StringVar(&cnf.GhostBinary, "binary", "gh-ost", "path to gh-ost binary") + flag.StringVar(&cnf.MysqlBinary, "mysql-binary", "mysql", "path to mysql binary") + flag.BoolVar(&printVersion, "version", false, "print version and exit") + flag.Parse() + + if printVersion { + fmt.Println(AppVersion) + os.Exit(0) + } + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true", + cnf.Username, + cnf.Password, + cnf.Host, + cnf.Port, + )) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + if err = localtests.WaitForMySQLAvailable(db); err != nil { + log.Fatalf("Failed to setup database client: %+v", err) + } + + var tests []localtests.Test + if testNoop { + tests = []localtests.Test{ + { + Name: "noop", + ExtraArgs: []string{ + `--alter='ENGINE=InnoDB'`, + }, + }, + } + } else { + tests, err = localtests.ReadTests(localtestsDir) + if err != nil { + log.Fatalf("Failed to read tests: %+v", err) + } + } + + for _, test := range tests { + if testName != "" && test.Name != testName { + continue + } + + log.Printf("Found test %q at %s/%s", test.Name, localtestsDir, test.Name) + if err = localtests.RunTest(db, cnf, test); err != nil { + log.Fatalf("Failed to run test %s: %+v", test.Name, err) + } + } +} diff --git a/go/localtests/localtests.go b/go/localtests/localtests.go new file mode 100644 index 000000000..4aad1c9e0 --- /dev/null +++ b/go/localtests/localtests.go @@ -0,0 +1,202 @@ +package localtests + +import ( + "database/sql" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/google/shlex" +) + +const ( + PrimaryHost = "primary" + DefaultHost = "replica" + DefaultPort int64 = 3306 + DefaultUsername = "gh-ost" + DefaultPassword = "gh-ost" + testDatabase = "test" + testTable = "gh_ost_test" + testSocketFile = "/tmp/gh-ost.test.sock" + throttleFlagFile = "/tmp/gh-ost-test.ghost.throttle.flag" + throttleQuery = "select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc" +) + +type Config struct { + Host string + Port int64 + Username string + Password string + GhostBinary string + MysqlBinary string +} + +type Test struct { + Name string + Path string + CreateSQLFile string + ExtraArgs []string + IgnoreVersions []string +} + +func WaitForMySQLAvailable(db *sql.DB) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-time.After(10 * time.Minute): + return errors.New("timed out waiting for mysql") + case <-ticker.C: + if err := db.Ping(); err != nil { + log.Println("Waiting for mysql to become available") + } else { + log.Println("MySQL is available") + return nil + } + } + } +} + +// Prepare runs a 'mysql' client/shell command to populate the test schema. +// The create.sql file is read by golang and passed to 'mysql' over stdin. +func (test *Test) Prepare(config Config) error { + if test.CreateSQLFile == "" { + return nil + } + + defaultsFile, err := writeMysqlClientDefaultsFile(config) + if err != nil { + return err + } + defer os.Remove(defaultsFile) + + flags := []string{ + fmt.Sprintf("--defaults-file=%s", defaultsFile), + fmt.Sprintf("--host=%s", PrimaryHost), // TODO: fix this + fmt.Sprintf("--port=%d", config.Port), + "--default-character-set=utf8mb4", + testDatabase, + } + log.Printf("[%s] running command: %s\n %s", test.Name, config.MysqlBinary, strings.Join(flags, "\n ")) + + createSQL, err := os.Open(test.CreateSQLFile) + if err != nil { + return err + } + defer createSQL.Close() + log.Printf("[%s] loaded sql from: %s", test.Name, test.CreateSQLFile) + + cmd := exec.Command(config.MysqlBinary, flags...) + cmd.Stdin = createSQL + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (test *Test) Migrate(db *sql.DB, config Config) error { + mysqlInfo, err := getMysqlHostInfo(db) + if err != nil { + return err + } + log.Printf("[%s] detected MySQL %s host %s:%d", test.Name, mysqlInfo.Version, config.Host, config.Port) + + flags := []string{ + fmt.Sprintf("--user=%s", config.Username), + fmt.Sprintf("--password=%s", config.Password), + fmt.Sprintf("--host=%s", config.Host), + fmt.Sprintf("--port=%d", config.Port), + fmt.Sprintf("--assume-master-host=primary:%d", mysqlInfo.Port), // TODO: fix this + fmt.Sprintf("--database=%s", testDatabase), + fmt.Sprintf("--table=%s", testTable), + "--assume-rbr", + "--chunk-size=10", + "--default-retries=3", + "--exact-rowcount", + "--initially-drop-old-table", + "--initially-drop-ghost-table", + "--initially-drop-socket-file", + fmt.Sprintf("--throttle-query=%s", throttleQuery), + fmt.Sprintf("--throttle-flag-file=%s", throttleFlagFile), + fmt.Sprintf("--serve-socket-file=%s", testSocketFile), + //"--test-on-replica", + "--allow-on-master", + "--debug", + "--execute", + "--stack", + "--verbose", + } + if len(test.ExtraArgs) > 0 { + flags = append(flags, test.ExtraArgs...) + } else { + flags = append(flags, `--alter='ENGINE=InnoDB'`) + } + + log.Printf("[%s] running gh-ost command: %s\n %s", test.Name, config.GhostBinary, strings.Join(flags, "\n ")) + cmd := exec.Command(config.GhostBinary, flags...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func ReadTests(testsDir string) (tests []Test, err error) { + subdirs, err := ioutil.ReadDir(testsDir) + if err != nil { + return tests, err + } + + for _, subdir := range subdirs { + test := Test{ + Name: subdir.Name(), + Path: filepath.Join(testsDir, subdir.Name()), + } + + stat, err := os.Stat(test.Path) + if err != nil || !stat.IsDir() { + continue + } + + test.CreateSQLFile = filepath.Join(test.Path, "create.sql") + if _, err = os.Stat(test.CreateSQLFile); err != nil { + log.Printf("Failed to find create.sql file %q: %+v", test.CreateSQLFile, err) + return tests, err + } + + extraArgsFile := filepath.Join(test.Path, "extra_args") + if _, err = os.Stat(extraArgsFile); err == nil { + extraArgsStr, err := readTestFile(extraArgsFile) + if err != nil { + log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) + return tests, err + } + if test.ExtraArgs, err = shlex.Split(extraArgsStr); err != nil { + log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) + return tests, err + } + } + + tests = append(tests, test) + } + + return tests, err +} + +func RunTest(db *sql.DB, config Config, test Test) error { + if err := test.Prepare(config); err != nil { + return err + } + log.Printf("[%s] prepared test", test.Name) + + if err := test.Migrate(db, config); err != nil { + return err + } + log.Printf("[%s] migrated test", test.Name) + + return nil +} diff --git a/go/localtests/utils.go b/go/localtests/utils.go new file mode 100644 index 000000000..7e2275aea --- /dev/null +++ b/go/localtests/utils.go @@ -0,0 +1,40 @@ +package localtests + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "strings" +) + +func readTestFile(file string) (string, error) { + bytes, err := ioutil.ReadFile(file) + return strings.TrimSpace(string(bytes)), err +} + +type MysqlInfo struct { + Host string + Port int64 + Version string +} + +func getMysqlHostInfo(db *sql.DB) (info MysqlInfo, err error) { + res := db.QueryRow("select @@hostname, @@port, @@version") + err = res.Scan(&info.Host, &info.Port, &info.Version) + return info, err +} + +func writeMysqlClientDefaultsFile(config Config) (string, error) { + defaultsFile, err := os.CreateTemp("", "gh-ost-localtests.my.cnf") + if err != nil { + return "", err + } + defer defaultsFile.Close() + + _, err = defaultsFile.Write([]byte(fmt.Sprintf( + "[client]\nuser=%s\npassword=%s\n", + config.Username, config.Password, + ))) + return defaultsFile.Name(), err +} diff --git a/go/sql/parser.go b/go/sql/parser.go index 2ddc60f50..7a0c7a67d 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -135,6 +135,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) { func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { this.alterStatementOptions = alterStatement + for _, trimQuote := range []string{`'`, `"`} { + if strings.HasPrefix(this.alterStatementOptions, trimQuote) && strings.HasSuffix(this.alterStatementOptions, trimQuote) { + this.alterStatementOptions = strings.TrimPrefix(this.alterStatementOptions, trimQuote) + this.alterStatementOptions = strings.TrimSuffix(this.alterStatementOptions, trimQuote) + } + } for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitSchema = submatch[1] diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index df9284280..18d8da2bd 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -6,6 +6,7 @@ package sql import ( + "fmt" "reflect" "testing" @@ -18,13 +19,40 @@ func init() { } func TestParseAlterStatement(t *testing.T) { - statement := "add column t int, engine=innodb" - parser := NewAlterTableParser() - err := parser.ParseAlterStatement(statement) - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(parser.alterStatementOptions, statement) - test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) - test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) + // plain alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) + } + // single-quoted alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } + // single-quoted w/comment alter + { + statement := "add column t int 'single-quoted comment'" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } + // double-quoted alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`"%s"`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } } func TestParseAlterStatementTrivialRename(t *testing.T) { diff --git a/localtests/Dockerfile b/localtests/Dockerfile new file mode 100644 index 000000000..47c44e9d1 --- /dev/null +++ b/localtests/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.17 +LABEL maintainer="github@github.com" + +RUN apt-get update +RUN apt-get install -y default-mysql-client lsb-release +RUN rm -rf /var/lib/apt/lists/* + +COPY . /go/src/github.com/github/gh-ost +WORKDIR /go/src/github.com/github/gh-ost + +RUN go build -o /usr/local/bin/gh-ost go/cmd/gh-ost/main.go +RUN go build -o /usr/local/bin/gh-ost-localtests go/cmd/gh-ost-localtests/main.go + +ENTRYPOINT ["gh-ost-localtests"] diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml new file mode 100644 index 000000000..45de01118 --- /dev/null +++ b/localtests/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.6" +services: + tests: + build: + context: ../ + dockerfile: localtests/Dockerfile + depends_on: + - primary + - replica + primary: + image: mysql:${MYSQL_TAG} + command: "--enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --server-id=1" + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + volumes: + - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" + replica: + image: mysql:${MYSQL_TAG} + command: "--enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --read-only=ON --server-id=2" + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: true + ports: + - "3306:3306" + volumes: + - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" + - "./init-replica.sql:/docker-entrypoint-initdb.d/02-init-replica.sql:ro" diff --git a/localtests/init-replica.sql b/localtests/init-replica.sql new file mode 100644 index 000000000..b48282ada --- /dev/null +++ b/localtests/init-replica.sql @@ -0,0 +1,9 @@ +STOP SLAVE; +RESET SLAVE; +RESET MASTER; + +CHANGE MASTER TO MASTER_HOST='primary', MASTER_USER='gh-ost', MASTER_PASSWORD='gh-ost', MASTER_PORT=3306, + MASTER_AUTO_POSITION=1, MASTER_CONNECT_RETRY=1; + +START SLAVE; +SET @@GLOBAL.read_only=ON; diff --git a/localtests/init.sql b/localtests/init.sql new file mode 100644 index 000000000..c5a1e4b46 --- /dev/null +++ b/localtests/init.sql @@ -0,0 +1,3 @@ +CREATE DATABASE IF NOT EXISTS `test`; + +GRANT ALL ON *.* TO `gh-ost`@`%` IDENTIFIED BY 'gh-ost'; diff --git a/localtests/keyword-column/extra_args b/localtests/keyword-column/extra_args index 5d73843b0..4b091d601 100644 --- a/localtests/keyword-column/extra_args +++ b/localtests/keyword-column/extra_args @@ -1 +1 @@ ---alter='add column `index` int unsigned' \ +--alter='add column `index` int unsigned' diff --git a/localtests/trivial/extra_args b/localtests/trivial/extra_args index 8b6320aa1..75bbe43a4 100644 --- a/localtests/trivial/extra_args +++ b/localtests/trivial/extra_args @@ -1 +1 @@ ---throttle-query='select false' \ +--throttle-query='select false' diff --git a/vendor/github.com/google/shlex/COPYING b/vendor/github.com/google/shlex/COPYING new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/google/shlex/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/shlex/README b/vendor/github.com/google/shlex/README new file mode 100644 index 000000000..c86bcc066 --- /dev/null +++ b/vendor/github.com/google/shlex/README @@ -0,0 +1,2 @@ +go-shlex is a simple lexer for go that supports shell-style quoting, +commenting, and escaping. diff --git a/vendor/github.com/google/shlex/shlex.go b/vendor/github.com/google/shlex/shlex.go new file mode 100644 index 000000000..d98308bce --- /dev/null +++ b/vendor/github.com/google/shlex/shlex.go @@ -0,0 +1,416 @@ +/* +Copyright 2012 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package shlex implements a simple lexer which splits input in to tokens using +shell-style rules for quoting and commenting. + +The basic use case uses the default ASCII lexer to split a string into sub-strings: + + shlex.Split("one \"two three\" four") -> []string{"one", "two three", "four"} + +To process a stream of strings: + + l := NewLexer(os.Stdin) + for ; token, err := l.Next(); err != nil { + // process token + } + +To access the raw token stream (which includes tokens for comments): + + t := NewTokenizer(os.Stdin) + for ; token, err := t.Next(); err != nil { + // process token + } + +*/ +package shlex + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// TokenType is a top-level token classification: A word, space, comment, unknown. +type TokenType int + +// runeTokenClass is the type of a UTF-8 character classification: A quote, space, escape. +type runeTokenClass int + +// the internal state used by the lexer state machine +type lexerState int + +// Token is a (type, value) pair representing a lexographical token. +type Token struct { + tokenType TokenType + value string +} + +// Equal reports whether tokens a, and b, are equal. +// Two tokens are equal if both their types and values are equal. A nil token can +// never be equal to another token. +func (a *Token) Equal(b *Token) bool { + if a == nil || b == nil { + return false + } + if a.tokenType != b.tokenType { + return false + } + return a.value == b.value +} + +// Named classes of UTF-8 runes +const ( + spaceRunes = " \t\r\n" + escapingQuoteRunes = `"` + nonEscapingQuoteRunes = "'" + escapeRunes = `\` + commentRunes = "#" +) + +// Classes of rune token +const ( + unknownRuneClass runeTokenClass = iota + spaceRuneClass + escapingQuoteRuneClass + nonEscapingQuoteRuneClass + escapeRuneClass + commentRuneClass + eofRuneClass +) + +// Classes of lexographic token +const ( + UnknownToken TokenType = iota + WordToken + SpaceToken + CommentToken +) + +// Lexer state machine states +const ( + startState lexerState = iota // no runes have been seen + inWordState // processing regular runes in a word + escapingState // we have just consumed an escape rune; the next rune is literal + escapingQuotedState // we have just consumed an escape rune within a quoted string + quotingEscapingState // we are within a quoted string that supports escaping ("...") + quotingState // we are within a string that does not support escaping ('...') + commentState // we are within a comment (everything following an unquoted or unescaped # +) + +// tokenClassifier is used for classifying rune characters. +type tokenClassifier map[rune]runeTokenClass + +func (typeMap tokenClassifier) addRuneClass(runes string, tokenType runeTokenClass) { + for _, runeChar := range runes { + typeMap[runeChar] = tokenType + } +} + +// newDefaultClassifier creates a new classifier for ASCII characters. +func newDefaultClassifier() tokenClassifier { + t := tokenClassifier{} + t.addRuneClass(spaceRunes, spaceRuneClass) + t.addRuneClass(escapingQuoteRunes, escapingQuoteRuneClass) + t.addRuneClass(nonEscapingQuoteRunes, nonEscapingQuoteRuneClass) + t.addRuneClass(escapeRunes, escapeRuneClass) + t.addRuneClass(commentRunes, commentRuneClass) + return t +} + +// ClassifyRune classifiees a rune +func (t tokenClassifier) ClassifyRune(runeVal rune) runeTokenClass { + return t[runeVal] +} + +// Lexer turns an input stream into a sequence of tokens. Whitespace and comments are skipped. +type Lexer Tokenizer + +// NewLexer creates a new lexer from an input stream. +func NewLexer(r io.Reader) *Lexer { + + return (*Lexer)(NewTokenizer(r)) +} + +// Next returns the next word, or an error. If there are no more words, +// the error will be io.EOF. +func (l *Lexer) Next() (string, error) { + for { + token, err := (*Tokenizer)(l).Next() + if err != nil { + return "", err + } + switch token.tokenType { + case WordToken: + return token.value, nil + case CommentToken: + // skip comments + default: + return "", fmt.Errorf("Unknown token type: %v", token.tokenType) + } + } +} + +// Tokenizer turns an input stream into a sequence of typed tokens +type Tokenizer struct { + input bufio.Reader + classifier tokenClassifier +} + +// NewTokenizer creates a new tokenizer from an input stream. +func NewTokenizer(r io.Reader) *Tokenizer { + input := bufio.NewReader(r) + classifier := newDefaultClassifier() + return &Tokenizer{ + input: *input, + classifier: classifier} +} + +// scanStream scans the stream for the next token using the internal state machine. +// It will panic if it encounters a rune which it does not know how to handle. +func (t *Tokenizer) scanStream() (*Token, error) { + state := startState + var tokenType TokenType + var value []rune + var nextRune rune + var nextRuneType runeTokenClass + var err error + + for { + nextRune, _, err = t.input.ReadRune() + nextRuneType = t.classifier.ClassifyRune(nextRune) + + if err == io.EOF { + nextRuneType = eofRuneClass + err = nil + } else if err != nil { + return nil, err + } + + switch state { + case startState: // no runes read yet + { + switch nextRuneType { + case eofRuneClass: + { + return nil, io.EOF + } + case spaceRuneClass: + { + } + case escapingQuoteRuneClass: + { + tokenType = WordToken + state = quotingEscapingState + } + case nonEscapingQuoteRuneClass: + { + tokenType = WordToken + state = quotingState + } + case escapeRuneClass: + { + tokenType = WordToken + state = escapingState + } + case commentRuneClass: + { + tokenType = CommentToken + state = commentState + } + default: + { + tokenType = WordToken + value = append(value, nextRune) + state = inWordState + } + } + } + case inWordState: // in a regular word + { + switch nextRuneType { + case eofRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case spaceRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case escapingQuoteRuneClass: + { + state = quotingEscapingState + } + case nonEscapingQuoteRuneClass: + { + state = quotingState + } + case escapeRuneClass: + { + state = escapingState + } + default: + { + value = append(value, nextRune) + } + } + } + case escapingState: // the rune after an escape character + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found after escape character") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + default: + { + state = inWordState + value = append(value, nextRune) + } + } + } + case escapingQuotedState: // the next rune after an escape character, in double quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found after escape character") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + default: + { + state = quotingEscapingState + value = append(value, nextRune) + } + } + } + case quotingEscapingState: // in escaping double quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found when expecting closing quote") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case escapingQuoteRuneClass: + { + state = inWordState + } + case escapeRuneClass: + { + state = escapingQuotedState + } + default: + { + value = append(value, nextRune) + } + } + } + case quotingState: // in non-escaping single quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found when expecting closing quote") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case nonEscapingQuoteRuneClass: + { + state = inWordState + } + default: + { + value = append(value, nextRune) + } + } + } + case commentState: // in a comment + { + switch nextRuneType { + case eofRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case spaceRuneClass: + { + if nextRune == '\n' { + state = startState + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } else { + value = append(value, nextRune) + } + } + default: + { + value = append(value, nextRune) + } + } + } + default: + { + return nil, fmt.Errorf("Unexpected state: %v", state) + } + } + } +} + +// Next returns the next token in the stream. +func (t *Tokenizer) Next() (*Token, error) { + return t.scanStream() +} + +// Split partitions a string into a slice of strings. +func Split(s string) ([]string, error) { + l := NewLexer(strings.NewReader(s)) + subStrings := make([]string, 0) + for { + word, err := l.Next() + if err != nil { + if err == io.EOF { + return subStrings, nil + } + return subStrings, err + } + subStrings = append(subStrings, word) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6f7639b77..2cfe1d69f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,6 +11,9 @@ github.com/go-mysql-org/go-mysql/utils # github.com/go-sql-driver/mysql v1.6.0 ## explicit; go 1.10 github.com/go-sql-driver/mysql +# github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +## explicit; go 1.13 +github.com/google/shlex # github.com/openark/golib v0.0.0-20210531070646-355f37940af8 ## explicit; go 1.16 github.com/openark/golib/log @@ -37,8 +40,6 @@ github.com/siddontang/go-log/loggers # go.uber.org/atomic v1.7.0 ## explicit; go 1.13 go.uber.org/atomic -# golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 -## explicit; go 1.11 # golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 ## explicit; go 1.11 golang.org/x/net/context