diff --git a/.ci b/.ci new file mode 160000 index 0000000000..5e83e8eac0 --- /dev/null +++ b/.ci @@ -0,0 +1 @@ +Subproject commit 5e83e8eac0fdb7ebb90945b745b348e882a68030 diff --git a/.ci/auto-gen.sh b/.ci/auto-gen.sh deleted file mode 100755 index c5bf00b2e8..0000000000 --- a/.ci/auto-gen.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -. "$(dirname $0)/variables.sh" - -autogen_clear() { - DIR="$1" - - rm -f ${DIR}/* -} - -mocks_clear() { - for DIR in $SRC; - do - MOCKS=${DIR}/*_mock.go - if ls $MOCKS &> /dev/null; then - for FILE in $(ls $MOCKS); - do - rm $FILE - done - fi - done -} - -autogen_cleanup() { - DIR="$1" - - FILES=${DIR}/*.go - for FILE in $(ls $FILES); - do - add_license $FILE $DIR - done -} - -mocks_cleanup() { - for DIR in $SRC; - do - MOCKS=${DIR}/*_mock.go - if ls $MOCKS &> /dev/null; then - for FILE in $(ls $MOCKS); - do - add_license $FILE $DIR - - # NB(xichen): there is an open issue (https://github.com/golang/mock/issues/30) - # with mockgen that causes the generated mock files to have vendored packages - # in the import list. For now we are working around it by removing the vendored - # path. Also sed -i'' does not work with BSD sed shipped with OS X, whereas - # sed -i '' doesn't work with GNU sed, so we work around it by redirecting to a - # temp file first and moving it back later. - sed "s|$VENDOR_PATH/||" $FILE > $FILE.tmp && mv $FILE.tmp $FILE - - # Strip GOPATH from the source file path - sed "s|Source: $GOPATH/src/\(.*\.go\)|Source: \1|" $FILE > $FILE.tmp && mv $FILE.tmp $FILE - done - fi - done -} - -add_license() { - FILE="$1" - DIR="$2" - - # Add uber license - PREV_PWD=$(pwd) - cd $DIR - $LICENSE_BIN --silent --file $(basename $FILE) - cd $PREV_PWD -} - -if [ $# -ne 2 ] || [ -z "$1" ] || [ -z "$2" ]; then - echo "usage: auto-gen.sh output_directory file_generation_rules_directory" - exit 1 -fi - -set -e - -. "$(dirname $0)/variables.sh" - -if [ "$2" = "generated/mocks" ]; then - mocks_clear -else - autogen_clear $1 -fi - -go generate $PACKAGE/$2 - -if [ "$2" = "generated/mocks" ]; then - mocks_cleanup -else - autogen_cleanup $1 -fi diff --git a/.ci/convert-test-data.sh b/.ci/convert-test-data.sh deleted file mode 100755 index e60a5ae3eb..0000000000 --- a/.ci/convert-test-data.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -. "$(dirname $0)/variables.sh" - -sed -i'' -e "s|$PACKAGE/||g" "$1" diff --git a/.ci/gotestcover/LICENSE b/.ci/gotestcover/LICENSE deleted file mode 100755 index 210d800c5e..0000000000 --- a/.ci/gotestcover/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (C) 2015 Pierre Durand - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/.ci/gotestcover/Makefile b/.ci/gotestcover/Makefile deleted file mode 100755 index 97e3b8528e..0000000000 --- a/.ci/gotestcover/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -#/bin/bash - -setup: - go get -u -v github.com/golang/lint/golint - go get -v -t ./... - -check: - gofmt -d . - go tool vet . - golint - -coverage: - gotestcover -coverprofile=coverage.txt github.com/pierrre/gotestcover - go tool cover -html=coverage.txt -o=coverage.html - -clean: - -rm coverage.txt - -rm coverage.html - gofmt -w . \ No newline at end of file diff --git a/.ci/gotestcover/README.md b/.ci/gotestcover/README.md deleted file mode 100755 index 5ee605a29b..0000000000 --- a/.ci/gotestcover/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Go test cover with multiple packages support - -## Features -- Coverage profile with multiple packages (`go test` doesn't support that) - -## Install -`go get github.com/pierrre/gotestcover` - -## Usage -```sh -gotestcover -coverprofile=cover.out mypackage -go tool cover -html=cover.out -o=cover.html -``` - -Run on multiple package with: -- `package1 package2` -- `package/...` - -Some `go test / build` flags are available. diff --git a/.ci/gotestcover/gotestcover.go b/.ci/gotestcover/gotestcover.go deleted file mode 100755 index 8a914b900e..0000000000 --- a/.ci/gotestcover/gotestcover.go +++ /dev/null @@ -1,282 +0,0 @@ -// Package gotestcover provides multiple packages support for Go test cover. -package main - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "io/ioutil" - "os" - "os/exec" - "runtime" - "strings" - "sync" -) - -var ( - // go build - flagA bool - flagX bool - flagRace bool - flagTags string - - // go test - flagV bool - flagCount int - flagCPU string - flagParallel string - flagRun string - flagShort bool - flagTimeout string - flagCoverMode string - flagCoverProfile string - - // custom - flagParallelPackages = runtime.GOMAXPROCS(0) - - // GAE/Go - flagGoogleAppEngine bool -) - -func main() { - err := run() - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func run() error { - err := parseFlags() - if err != nil { - return err - } - pkgArgs, flagArgs := parseArgs() - pkgs, err := resolvePackages(pkgArgs) - if err != nil { - return err - } - cov, failed := runAllPackageTests(pkgs, flagArgs, func(out string) { - fmt.Print(out) - }) - err = writeCoverProfile(cov) - if err != nil { - return err - } - if failed { - return fmt.Errorf("test failed") - } - return nil -} - -func parseFlags() error { - flag.BoolVar(&flagA, "a", flagA, "see 'go build' help") - flag.BoolVar(&flagX, "x", flagX, "see 'go build' help") - flag.BoolVar(&flagRace, "race", flagRace, "see 'go build' help") - flag.StringVar(&flagTags, "tags", flagTags, "see 'go build' help") - - flag.BoolVar(&flagV, "v", flagV, "see 'go test' help") - flag.IntVar(&flagCount, "count", flagCount, "see 'go test' help") - flag.StringVar(&flagCPU, "cpu", flagCPU, "see 'go test' help") - flag.StringVar(&flagParallel, "parallel", flagParallel, "see 'go test' help") - flag.StringVar(&flagRun, "run", flagRun, "see 'go test' help") - flag.BoolVar(&flagShort, "short", flagShort, "see 'go test' help") - flag.StringVar(&flagTimeout, "timeout", flagTimeout, "see 'go test' help") - flag.StringVar(&flagCoverMode, "covermode", flagCoverMode, "see 'go test' help") - flag.StringVar(&flagCoverProfile, "coverprofile", flagCoverProfile, "see 'go test' help") - - flag.IntVar(&flagParallelPackages, "parallelpackages", flagParallelPackages, "Number of package test run in parallel") - - flag.BoolVar(&flagGoogleAppEngine, "gae", flagGoogleAppEngine, "Bool of Command exec in GAE/Go") - - flag.Parse() - if flagCoverProfile == "" { - return fmt.Errorf("flag coverprofile must be set") - } - if flagParallelPackages < 1 { - return fmt.Errorf("flag parallelpackages must be greater than or equal to 1") - } - return nil -} - -func parseArgs() (pkgArgs, flagArgs []string) { - args := flag.Args() - for i, a := range args { - if strings.HasPrefix(a, "-") { - return args[:i], args[i:] - } - } - return args, nil -} - -func resolvePackages(pkgArgs []string) ([]string, error) { - cmdArgs := []string{"list"} - cmdArgs = append(cmdArgs, pkgArgs...) - cmdOut, err := runGoCommand(cmdArgs...) - if err != nil { - return nil, err - } - var pkgs []string - sc := bufio.NewScanner(bytes.NewReader(cmdOut)) - for sc.Scan() { - pkgs = append(pkgs, sc.Text()) - } - return pkgs, nil -} - -func runAllPackageTests(pkgs []string, flgs []string, pf func(string)) ([]byte, bool) { - pkgch := make(chan string) - type res struct { - out string - cov []byte - err error - } - resch := make(chan res) - wg := new(sync.WaitGroup) - wg.Add(flagParallelPackages) - go func() { - for _, pkg := range pkgs { - pkgch <- pkg - } - close(pkgch) - wg.Wait() - close(resch) - }() - for i := 0; i < flagParallelPackages; i++ { - go func() { - for p := range pkgch { - out, cov, err := runPackageTests(p, flgs) - resch <- res{ - out: out, - cov: cov, - err: err, - } - } - wg.Done() - }() - } - failed := false - var cov []byte - for r := range resch { - if r.err == nil { - pf(r.out) - cov = append(cov, r.cov...) - } else { - pf(r.err.Error()) - failed = true - } - } - return cov, failed -} - -func runPackageTests(pkg string, flgs []string) (out string, cov []byte, err error) { - coverFile, err := ioutil.TempFile("", "gotestcover-") - if err != nil { - return "", nil, err - } - defer os.Remove(coverFile.Name()) - defer coverFile.Close() - var args []string - args = append(args, "test") - - if flagA { - args = append(args, "-a") - } - if flagX { - args = append(args, "-x") - } - if flagRace { - args = append(args, "-race") - } - if flagTags != "" { - args = append(args, "-tags", flagTags) - } - - if flagV { - args = append(args, "-v") - } - if flagCount != 0 { - args = append(args, "-count", fmt.Sprint(flagCount)) - } - if flagCPU != "" { - args = append(args, "-cpu", flagCPU) - } - if flagParallel != "" { - args = append(args, "-parallel", flagParallel) - } - if flagRun != "" { - args = append(args, "-run", flagRun) - } - if flagShort { - args = append(args, "-short") - } - if flagTimeout != "" { - args = append(args, "-timeout", flagTimeout) - } - args = append(args, "-cover") - if flagCoverMode != "" { - args = append(args, "-covermode", flagCoverMode) - } - args = append(args, "-coverprofile", coverFile.Name()) - - args = append(args, pkg) - - args = append(args, flgs...) - - cmdOut, err := runGoCommand(args...) - if err != nil { - return "", nil, err - } - cov, err = ioutil.ReadAll(coverFile) - if err != nil { - return "", nil, err - } - cov = removeFirstLine(cov) - return string(cmdOut), cov, nil -} - -func writeCoverProfile(cov []byte) error { - if len(cov) == 0 { - return nil - } - buf := new(bytes.Buffer) - mode := flagCoverMode - if mode == "" { - if flagRace { - mode = "atomic" - } else { - mode = "set" - } - } - fmt.Fprintf(buf, "mode: %s\n", mode) - buf.Write(cov) - return ioutil.WriteFile(flagCoverProfile, buf.Bytes(), os.FileMode(0644)) -} - -func runGoCommand(args ...string) ([]byte, error) { - goCmd := "go" - if flagGoogleAppEngine { - goCmd = "goapp" - } - cmd := exec.Command(goCmd, args...) - out, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("command %s: %s\n%s", cmd.Args, err, out) - } - return out, nil -} - -func removeFirstLine(b []byte) []byte { - out := new(bytes.Buffer) - sc := bufio.NewScanner(bytes.NewReader(b)) - firstLine := true - for sc.Scan() { - if firstLine { - firstLine = false - continue - } - fmt.Fprintf(out, "%s\n", sc.Bytes()) - } - return out.Bytes() -} diff --git a/.ci/gotestcover/gotestcover_test.go b/.ci/gotestcover/gotestcover_test.go deleted file mode 100755 index 3a662d6968..0000000000 --- a/.ci/gotestcover/gotestcover_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "os" - "testing" -) - -func TestParseFlags(t *testing.T) { - os.Args = []string{"gotestcover", - "-v", - "-a", - "-x", - "-tags=foobar", - "-race", - "-cpu=4", - "-parallel=2", - "-run=abc", - "-short", - "-timeout=15s", - "-covermode=atomic", - "-parallelpackages=2", - "-coverprofile=cover.out", - "-gae", - } - - err := parseFlags() - - if err != nil { - t.Fatal(err) - } - - if !flagV { - t.Errorf("flagV should be set to true") - } - - if !flagA { - t.Errorf("flagA should be set to true") - } - - if !flagX { - t.Errorf("flagX should be set to true") - } - - if flagTags != "foobar" { - t.Errorf("flagCPU is not equal to foobar, got %s", flagTags) - } - - if !flagRace { - t.Errorf("flagRace should be set to true") - } - - if flagCPU != "4" { - t.Errorf("flagCPU is not equal to 4, got %s", flagCPU) - } - - if flagParallel != "2" { - t.Errorf("flagParallel is not equal to 2, got %s", flagParallel) - } - - if flagRun != "abc" { - t.Errorf("flagRun is not equal to 'abc', got %s", flagRun) - } - - if !flagShort { - t.Errorf("flagShort should be set to true") - } - - if flagTimeout != "15s" { - t.Errorf("flagTimeout is not equal to '15s', got %s", flagTimeout) - } - - if flagCoverMode != "atomic" { - t.Errorf("flagCoverMode is not equal to 'atomic', got %s", flagCoverMode) - } - - if flagParallelPackages != 2 { - t.Errorf("flagParallelPackages is not equal to '2', got %d", flagParallelPackages) - } - - if flagCoverProfile != "cover.out" { - t.Errorf("flagCoverProfile is not equal to 'cover.out', got %s", flagCoverProfile) - } - - if !flagGoogleAppEngine { - t.Errorf("flagGoogleAppEngine should be set to true") - } -} diff --git a/.ci/lint.sh b/.ci/lint.sh deleted file mode 100755 index 33f31bcef4..0000000000 --- a/.ci/lint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -! golint ./... | egrep -v $(cat .excludelint | sed -e 's/^/ -e /' | sed -e 's/(//' | sed -e 's/)//' | tr -d '\n') diff --git a/.ci/test-cover.sh b/.ci/test-cover.sh deleted file mode 100755 index 62265c295b..0000000000 --- a/.ci/test-cover.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -. "$(dirname $0)/variables.sh" - -set -e - -TARGET=${1:-profile.cov} -LOG=${2:-test.log} - -rm $TARGET &>/dev/null || true -echo "mode: count" > $TARGET -echo "" > $LOG - -DIRS="" -for DIR in $SRC; -do - if ls $DIR/*_test.go &> /dev/null; then - DIRS="$DIRS $DIR" - fi -done - -NPROC=$(getconf _NPROCESSORS_ONLN) -echo "test-cover begin: concurrency $NPROC" -go run .ci/gotestcover/gotestcover.go -race -covermode=atomic -coverprofile=profile.tmp -v -parallelpackages $NPROC $DIRS | tee $LOG - -TEST_EXIT=${PIPESTATUS[0]} - -cat profile.tmp | grep -v "_mock.go" > $TARGET - -find . -not -path '*/vendor/*' | grep \\.tmp$ | xargs -I{} rm {} -echo "test-cover result: $TEST_EXIT" - -exit $TEST_EXIT diff --git a/.ci/test-integration.sh b/.ci/test-integration.sh deleted file mode 100755 index 0c3caa63ad..0000000000 --- a/.ci/test-integration.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -. "$(dirname $0)/variables.sh" - -set -e - -TAGS="integration" -DIR="integration" - -# compile the integration test binary -go test -test.c -test.tags=${TAGS} ./${DIR} - -# list the tests -TESTS=$(./integration.test -test.v -test.short | grep RUN | tr -s " " | cut -d ' ' -f 3) - -# execute tests one by one for isolation -for TEST in $TESTS; do - ./integration.test -test.v -test.run $TEST ./integration - TEST_EXIT=$? - if [ "$TEST_EXIT" != "0" ]; then - echo "$TEST failed" - exit $TEST_EXIT - fi - sleep 0.1 -done - -echo "PASS all integrations tests" diff --git a/.ci/uber-licence b/.ci/uber-licence deleted file mode 160000 index e35b7f99af..0000000000 --- a/.ci/uber-licence +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e35b7f99af2d110505bc81d8a9449dafaa326730 diff --git a/.ci/variables.sh b/.ci/variables.sh deleted file mode 100755 index 62be10b3f6..0000000000 --- a/.ci/variables.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -export PACKAGE='github.com/m3db/m3db' -export VENDOR_PATH=$PACKAGE/vendor -export LICENSE_BIN=$GOPATH/src/$PACKAGE/.ci/uber-licence/bin/licence -export GO15VENDOREXPERIMENT=1 -export SRC=$(find ./ -maxdepth 10 -not -path '*/.git*' -not -path '*/.ci*' -not -path '*/_*' -not -path '*/vendor/*' -type d) diff --git a/.gitmodules b/.gitmodules index 461d0cb904..b9973c5add 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule ".ci/uber-licence"] - path = .ci/uber-licence - url = https://github.com/uber/uber-licence +[submodule ".ci"] + path = .ci + url = https://github.com/m3db/ci-scripts.git diff --git a/.travis.yml b/.travis.yml index 3677f63810..d89d5603d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ install: make install-ci env: # Set higher timeouts for Travis - TEST_TIMEOUT_SCALE=20 + - PACKAGE=github.com/m3db/m3db sudo: required -dist: trusty +dist: trusty script: - make lint - make test-ci-unit