From 0f25c42e2e3fdd7a5a7b5f8ab664b5a804f2e936 Mon Sep 17 00:00:00 2001 From: Sven Walter Date: Thu, 16 Aug 2018 10:20:46 +0200 Subject: [PATCH] add Jsonnet importer --- Gopkg.lock | 14 ++-- Gopkg.toml | 2 +- pkg/api/render.go | 12 ++-- pkg/api/render_test.go | 34 +++++++-- pkg/gh/client.go | 14 ++-- pkg/gh/fake/impl.go | 6 +- pkg/gh/fake/package_test.go | 16 ++--- pkg/gh/fake/utils.go | 2 +- pkg/gh/file.go | 6 +- pkg/gh/importer.go | 61 ++++++++++++++++ pkg/gh/importer_test.go | 136 ++++++++++++++++++++++++++++++++++++ pkg/gh/location.go | 16 ++++- pkg/settings/settings.go | 4 ++ 13 files changed, 280 insertions(+), 43 deletions(-) create mode 100644 pkg/gh/importer.go create mode 100644 pkg/gh/importer_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 322da01..fe304e2 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -62,7 +62,7 @@ branch = "master" name = "github.com/google/btree" packages = ["."] - revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" + revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" [[projects]] name = "github.com/google/go-github" @@ -77,8 +77,8 @@ "ast", "parser" ] - revision = "dfddf2b4e3aec377b0dcdf247ff92e7d078b8179" - version = "v0.10.0" + revision = "6144c57d2a054b72ff46219f655bf863b940e174" + version = "v0.11.2" [[projects]] branch = "master" @@ -255,7 +255,7 @@ branch = "master" name = "github.com/spf13/jwalterweatherman" packages = ["."] - revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + revision = "14d3d4c518341bea657dd8a226f5121c0ff8c9f2" [[projects]] name = "github.com/spf13/pflag" @@ -286,7 +286,7 @@ "http2/hpack", "idna" ] - revision = "f9ce57c11b242f0f1599cf25c89d8cb02c45295a" + revision = "c39426892332e1bb5ec0a434a079bf82f5d30c54" [[projects]] branch = "master" @@ -304,7 +304,7 @@ "unix", "windows" ] - revision = "f0d5e33068cb57c22a181f5df0ffda885309eb5a" + revision = "14742f9018cd6651ec7364dc6ee08af0baaa1031" [[projects]] name = "golang.org/x/text" @@ -514,6 +514,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "83bba37b06273ef571e890f3d49ce4711ddc90c251e9d2f7ee508b8a61b705d0" + inputs-digest = "19a20a55c50c1a3aa4567b55bb18425454067a9437bb9a8e4545034ae7f705bc" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a014e97..96cb089 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -68,4 +68,4 @@ [[constraint]] name = "github.com/google/go-jsonnet" - version = "^0.10.0" + version = "^0.11.2" diff --git a/pkg/api/render.go b/pkg/api/render.go index e45ffbd..6a17a1e 100644 --- a/pkg/api/render.go +++ b/pkg/api/render.go @@ -33,6 +33,10 @@ func (app *App) decode(files []gh.File, vars templates.Variables) ([]runtime.Obj var objects []runtime.Object for _, file := range files { + log.WithFields(log.Fields{ + "Location": file.Location, + }).Debug("decoding file") + switch { case strings.HasSuffix(file.Name(), ".yml"): fallthrough @@ -97,11 +101,7 @@ func (app *App) decodeYAML(file gh.File, vars templates.Variables) ([]runtime.Ob func (app *App) decodeJsonnet(file gh.File, vars templates.Variables, all []gh.File) ([]runtime.Object, error) { var objects []runtime.Object - importer := new(jsonnet.MemoryImporter) - importer.Data = make(map[string]string) - for _, f := range all { - importer.Data[f.Name()] = f.Content - } + importer := gh.NewJsonnetImporter(app.Clients.GitHub) vm := jsonnet.MakeVM() vm.Importer(importer) @@ -109,7 +109,7 @@ func (app *App) decodeJsonnet(file gh.File, vars templates.Variables, all []gh.F vm.ExtVar(k, v) } - docs, err := vm.EvaluateSnippetStream(file.Name(), file.Content) + docs, err := vm.EvaluateSnippetStream(file.Location.String(), file.Content) if err != nil { return nil, errors.WithStack(err) } diff --git a/pkg/api/render_test.go b/pkg/api/render_test.go index cb60f3c..0d024f2 100644 --- a/pkg/api/render_test.go +++ b/pkg/api/render_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/rebuy-de/kubernetes-deployment/pkg/gh" + fakeGH "github.com/rebuy-de/kubernetes-deployment/pkg/gh/fake" "github.com/rebuy-de/kubernetes-deployment/pkg/interceptors" "github.com/rebuy-de/kubernetes-deployment/pkg/templates" "github.com/rebuy-de/rebuy-go-sdk/testutil" @@ -73,18 +74,43 @@ func TestDecode(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - files := []gh.File{} + files := fakeGH.Files{} for _, fname := range tc.files { files = append(files, gh.File{ - Path: fname, + Location: &gh.Location{ + Owner: "rebuy-de", + Repo: "test", + Path: fname, + Ref: "master", + }, Content: readFile(t, path.Join("test-fixtures", fname)), }) } - app := App{Interceptors: &interceptors.Multi{}} + github := &fakeGH.GitHub{ + "rebuy-de": fakeGH.Repos{ + "test": fakeGH.Branches{ + "master": fakeGH.Branch{ + Meta: gh.Branch{ + Name: "master", + SHA: "5a5369823a2a9a6ad9c241b404be39f802d41d48", + }, + Files: files, + }, + }, + }, + } + + app := App{ + Interceptors: &interceptors.Multi{}, + Clients: &Clients{ + GitHub: github, + }, + } + objects, err := app.decode(files, vars) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } g := path.Join("test-fixtures", fmt.Sprintf("render-golden-%s.json", tc.name)) testutil.AssertGoldenJSON(t, g, objects) diff --git a/pkg/gh/client.go b/pkg/gh/client.go index 196dc8a..ab1b64a 100644 --- a/pkg/gh/client.go +++ b/pkg/gh/client.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "path" - "regexp" "strings" "time" @@ -20,10 +19,6 @@ import ( log "github.com/sirupsen/logrus" ) -var ( - ContentLocationRE = regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*)$`) -) - type Interface interface { GetBranch(location *Location) (*Branch, error) GetFile(location *Location) (File, error) @@ -186,7 +181,7 @@ func (gh *API) GetFile(location *Location) (File, error) { gh.statsd.Gauge("github.rate.remaining", resp.Rate.Remaining) - return File{location.Path, content}, nil + return File{Location: location, Content: content}, nil } func (gh *API) GetFiles(location *Location) ([]File, error) { @@ -244,11 +239,16 @@ func (gh *API) GetFiles(location *Location) ([]File, error) { _, notAFile := err.(ErrNotAFile) if !notAFile { return nil, errors.Wrapf(err, - "unable to decode file '%v'", + "unable to download file '%v'", location) } + log.WithFields(log.Fields{ + "Location": file.Location, + }).Debug("skipping path, because it is not a file") + continue } + files = append(files, file) } diff --git a/pkg/gh/fake/impl.go b/pkg/gh/fake/impl.go index b38159c..4b2d1cb 100644 --- a/pkg/gh/fake/impl.go +++ b/pkg/gh/fake/impl.go @@ -57,18 +57,18 @@ func (d *GitHub) GetBranch(l *gh.Location) (*gh.Branch, error) { func (d *GitHub) GetFile(l *gh.Location) (gh.File, error) { for _, file := range (*d)[l.Owner][l.Repo][l.Ref].Files { - if file.Path == l.Path { + if file.Location.Path == l.Path { return file, nil } } - return gh.File{}, nil + return gh.File{}, fmt.Errorf("File %s not found", l.String()) } func (d *GitHub) GetFiles(l *gh.Location) ([]gh.File, error) { var files []gh.File for _, file := range (*d)[l.Owner][l.Repo][l.Ref].Files { - dir, _ := path.Split("/" + file.Path) + dir, _ := path.Split("/" + file.Location.Path) if path.Clean("/"+dir+"/") == path.Clean("/"+l.Path+"/") { files = append(files, file) } diff --git a/pkg/gh/fake/package_test.go b/pkg/gh/fake/package_test.go index 5dc1d49..fb61c4d 100644 --- a/pkg/gh/fake/package_test.go +++ b/pkg/gh/fake/package_test.go @@ -22,10 +22,10 @@ var ( "master": Branch{ Meta: ExampleBranch, Files: Files{ - {Path: "deployments.yaml", Content: YAML([]string{"foo", "bar"})}, - {Path: "README.md", Content: "blubber"}, - {Path: "sub/foo.txt", Content: "bar"}, - {Path: "sub/bim.txt", Content: "baz"}, + {Location: &gh.Location{Path: "deployments.yaml"}, Content: YAML([]string{"foo", "bar"})}, + {Location: &gh.Location{Path: "README.md"}, Content: "blubber"}, + {Location: &gh.Location{Path: "sub/foo.txt"}, Content: "bar"}, + {Location: &gh.Location{Path: "sub/bim.txt"}, Content: "baz"}, }, }, }, @@ -89,8 +89,8 @@ func TestGetFiles(t *testing.T) { } expected := []gh.File{ - {Path: "deployments.yaml", Content: "- foo\n- bar\n"}, - {Path: "README.md", Content: "blubber"}, + {Location: &gh.Location{Path: "deployments.yaml"}, Content: "- foo\n- bar\n"}, + {Location: &gh.Location{Path: "README.md"}, Content: "blubber"}, } if !reflect.DeepEqual(files, expected) { @@ -107,8 +107,8 @@ func TestGetSubdirectoryFiles(t *testing.T) { } expected := []gh.File{ - {Path: "sub/foo.txt", Content: "bar"}, - {Path: "sub/bim.txt", Content: "baz"}, + {Location: &gh.Location{Path: "sub/foo.txt"}, Content: "bar"}, + {Location: &gh.Location{Path: "sub/bim.txt"}, Content: "baz"}, } if !reflect.DeepEqual(files, expected) { diff --git a/pkg/gh/fake/utils.go b/pkg/gh/fake/utils.go index 0e8f36e..c894085 100644 --- a/pkg/gh/fake/utils.go +++ b/pkg/gh/fake/utils.go @@ -36,7 +36,7 @@ func ScanFiles(root string) Files { } raw, err := ioutil.ReadFile(path) - files = append(files, gh.File{Path: relPath, Content: string(raw)}) + files = append(files, gh.File{Location: &gh.Location{Path: relPath}, Content: string(raw)}) return err }) if err != nil { diff --git a/pkg/gh/file.go b/pkg/gh/file.go index 8f9dec6..8998098 100644 --- a/pkg/gh/file.go +++ b/pkg/gh/file.go @@ -3,11 +3,11 @@ package gh import "path" type File struct { - Path string - Content string + Location *Location + Content string } func (f *File) Name() string { - _, name := path.Split(f.Path) + _, name := path.Split(f.Location.Path) return name } diff --git a/pkg/gh/importer.go b/pkg/gh/importer.go new file mode 100644 index 0000000..6d28ac6 --- /dev/null +++ b/pkg/gh/importer.go @@ -0,0 +1,61 @@ +package gh + +import ( + "path" + "path/filepath" + "strings" + + jsonnet "github.com/google/go-jsonnet" +) + +type jsonnetImporter struct { + client Interface +} + +func NewJsonnetImporter(client Interface) jsonnet.Importer { + return &jsonnetImporter{ + client: client, + } +} + +func (i *jsonnetImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { + fromLocation, err := NewLocation(importedFrom) + if err != nil { + return jsonnet.MakeContents(""), "", err + } + + var location *Location + if strings.HasPrefix(importedPath, "github.com/") { + // location constrution for absolute paths + location, err = NewLocation(importedPath) + if err != nil { + return jsonnet.MakeContents(""), "", err + } + } else { + // location constrution for relative paths + fromDir := filepath.Dir(fromLocation.String()) + absolute := path.Clean(path.Join(fromDir, importedPath)) + + location, err = NewLocation(absolute) + if err != nil { + return jsonnet.MakeContents(""), "", err + } + } + + // having no ref means same ref, if inside same repo and "master" if it is in another repo + if location.Ref == "" { + if location.Owner == fromLocation.Owner && location.Repo == fromLocation.Repo { + location.Ref = fromLocation.Ref + } else { + location.Ref = "master" + } + + } + + file, err := i.client.GetFile(location) + if err != nil { + return jsonnet.MakeContents(""), "", err + } + + return jsonnet.MakeContents(file.Content), location.String(), nil +} diff --git a/pkg/gh/importer_test.go b/pkg/gh/importer_test.go new file mode 100644 index 0000000..9de79b8 --- /dev/null +++ b/pkg/gh/importer_test.go @@ -0,0 +1,136 @@ +package gh_test + +import ( + "testing" + "time" + + "github.com/rebuy-de/kubernetes-deployment/pkg/gh" + "github.com/rebuy-de/kubernetes-deployment/pkg/gh/fake" +) + +func TestImporter(t *testing.T) { + client := &fake.GitHub{ + "rebuy-de": fake.Repos{ + "web": fake.Branches{ + "master": fake.Branch{ + Meta: gh.Branch{ + SHA: "bacdb99030b908fd853367f3c5cbbe20aa424672", + Author: "Hubot", + Date: time.Now(), + Message: "fix the fix", + }, + Files: fake.Files{ + {Location: &gh.Location{Path: ".deployment/ingress.libsonnet"}, Content: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@master"}, + {Location: &gh.Location{Path: "blue/deployment/k8s/util.libsonnet"}, Content: "github.com/rebuy-de/web/blue/deployment/k8s/util.libsonnet@master"}, + }, + }, + "cloud-1337": fake.Branch{ + Meta: gh.Branch{ + SHA: "2b8f82e0e1027b4c9f56da5a6175a099c9827859", + Author: "test", + Date: time.Now(), + Message: "dis is test", + }, + Files: fake.Files{ + {Location: &gh.Location{Path: ".deployment/ingress.libsonnet"}, Content: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@cloud-1337"}, + }, + }, + }, + "jsonnet-libraries": fake.Branches{ + "master": fake.Branch{ + Meta: gh.Branch{ + SHA: "15f088d8d78544cb4a900b28b8a2dcb28a130a3f", + Author: "root", + Date: time.Now(), + Message: "initial", + }, + Files: fake.Files{ + {Location: &gh.Location{Path: "util.libsonnet"}, Content: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet@master"}, + }, + }, + "cloud-42": fake.Branch{ + Meta: gh.Branch{ + SHA: "e95f454ee4b24573e6a38186d97edfa90a01f3c3", + Author: "user", + Date: time.Now(), + Message: "add flux", + }, + Files: fake.Files{ + {Location: &gh.Location{Path: "util.libsonnet"}, Content: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet@cloud-42"}, + }, + }, + }, + }, + } + + importer := gh.NewJsonnetImporter(client) + + cases := []struct { + name string + importedFrom string + importedPath string + foundAt string + }{ + { + name: "same_repo_no_ref_relative_path", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "../../../.deployment/ingress.libsonnet", + foundAt: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@cloud-1337", + }, + { + name: "same_repo_no_ref_absolute_path", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "github.com/rebuy-de/web/.deployment/ingress.libsonnet", + foundAt: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@cloud-1337", + }, + { + name: "same_repo_same_ref_relative_path", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "../../../.deployment/ingress.libsonnet@cloud-1337", + foundAt: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@cloud-1337", + }, + { + name: "same_repo_other_ref_relative_path", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "../../../.deployment/ingress.libsonnet@master", + foundAt: "github.com/rebuy-de/web/.deployment/ingress.libsonnet@master", + }, + { + name: "other_repo_no_ref", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet", + foundAt: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet@master", + }, + { + name: "other_repo_with_ref", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@cloud-1337", + importedPath: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet@cloud-42", + foundAt: "github.com/rebuy-de/jsonnet-libraries/util.libsonnet@cloud-42", + }, + { + name: "local_import", + importedFrom: "github.com/rebuy-de/web/blue/deployment/k8s/ingress.jsonnet@master", + importedPath: "util.libsonnet", + foundAt: "github.com/rebuy-de/web/blue/deployment/k8s/util.libsonnet@master", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + contents, foundAt, err := importer.Import(tc.importedFrom, tc.importedPath) + if err != nil { + t.Error(err) + return + } + + if foundAt != tc.foundAt { + t.Errorf("%s != %s", foundAt, tc.foundAt) + } + + if contents.String() != tc.foundAt { + t.Errorf("%s != %s", contents.String(), tc.foundAt) + } + }) + } + +} diff --git a/pkg/gh/location.go b/pkg/gh/location.go index 293f99a..dc92043 100644 --- a/pkg/gh/location.go +++ b/pkg/gh/location.go @@ -3,12 +3,17 @@ package gh import ( "fmt" "path/filepath" + "regexp" "strings" "github.com/imdario/mergo" "github.com/pkg/errors" ) +var ( + ContentLocationRE = regexp.MustCompile(`^github.com/([^/]+)/([^/]+)/(.*?)(@([^@]+))?$`) +) + type Location struct { Owner, Repo, Path, Ref string `yaml:",omitempty"` } @@ -17,19 +22,24 @@ func NewLocation(location string) (*Location, error) { matches := ContentLocationRE.FindStringSubmatch(location) if matches == nil { return nil, errors.Errorf( - "GitHub location must have the form `github.com/:owner:/:repo:/:path:`") + "GitHub location must have the form `github.com/:owner:/:repo:/:path:[@:ref:]`, but got `%s`", + location) } return &Location{ Owner: matches[1], Repo: matches[2], Path: matches[3], - Ref: "master", + Ref: matches[5], }, nil } func (l Location) String() string { - return fmt.Sprintf("github.com/%s/%s/%s@%s", l.Owner, l.Repo, l.Path, l.Ref) + path := fmt.Sprintf("github.com/%s/%s/%s", l.Owner, l.Repo, l.Path) + if l.Ref != "" { + path = fmt.Sprintf("%s@%s", path, l.Ref) + } + return path } func (l *Location) Defaults(defaults Location) { diff --git a/pkg/settings/settings.go b/pkg/settings/settings.go index 81915b5..e0b6fe4 100644 --- a/pkg/settings/settings.go +++ b/pkg/settings/settings.go @@ -56,6 +56,10 @@ func ReadFromGitHub(filename string, client gh.Interface) (*Settings, error) { return nil, errors.Wrapf(err, "parse GitHub location '%s'; use './' prefix to use a directory named 'github.com'", filename) } + location.Defaults(gh.Location{ + Ref: "master", + }) + file, err := client.GetFile(location) if err != nil { return nil, errors.Wrapf(err, "could not download file '%s'", location)