From 964630d087ef0ce0ff432c6f5f02b5b5d605ac0f Mon Sep 17 00:00:00 2001 From: Hongkai Liu Date: Wed, 21 Aug 2019 11:16:37 -0400 Subject: [PATCH] Publicize functions in the autobumper This enables reusing the functions as API for autobumping the prow-images for other prow deployment, eg, [1]. [1]. https://github.com/openshift/release --- experiment/autobumper/BUILD.bazel | 8 +- experiment/autobumper/bumper/BUILD.bazel | 28 +++ experiment/autobumper/bumper/bumper.go | 219 +++++++++++++++++++++++ experiment/autobumper/main.go | 194 ++------------------ 4 files changed, 265 insertions(+), 184 deletions(-) create mode 100644 experiment/autobumper/bumper/BUILD.bazel create mode 100644 experiment/autobumper/bumper/bumper.go diff --git a/experiment/autobumper/BUILD.bazel b/experiment/autobumper/BUILD.bazel index 6c83127eca5e..d7d0cd42654c 100644 --- a/experiment/autobumper/BUILD.bazel +++ b/experiment/autobumper/BUILD.bazel @@ -6,10 +6,9 @@ go_library( importpath = "k8s.io/test-infra/experiment/autobumper", visibility = ["//visibility:private"], deps = [ - "//experiment/image-bumper/bumper:go_default_library", + "//experiment/autobumper/bumper:go_default_library", "//prow/config/secret:go_default_library", "//prow/github:go_default_library", - "//robots/pr-creator/updater:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", ], ) @@ -29,7 +28,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//experiment/autobumper/bumper:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/experiment/autobumper/bumper/BUILD.bazel b/experiment/autobumper/bumper/BUILD.bazel new file mode 100644 index 000000000000..7bdeb1985e7b --- /dev/null +++ b/experiment/autobumper/bumper/BUILD.bazel @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["bumper.go"], + importpath = "k8s.io/test-infra/experiment/autobumper/bumper", + visibility = ["//visibility:public"], + deps = [ + "//experiment/image-bumper/bumper:go_default_library", + "//prow/github:go_default_library", + "//robots/pr-creator/updater:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/experiment/autobumper/bumper/bumper.go b/experiment/autobumper/bumper/bumper.go new file mode 100644 index 000000000000..35ffbab6120f --- /dev/null +++ b/experiment/autobumper/bumper/bumper.go @@ -0,0 +1,219 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bumper + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/sirupsen/logrus" + + "k8s.io/test-infra/experiment/image-bumper/bumper" + "k8s.io/test-infra/prow/github" + "k8s.io/test-infra/robots/pr-creator/updater" +) + +const ( + prowPrefix = "gcr.io/k8s-prow/" + testImagePrefix = "gcr.io/k8s-testimages/" + prowRepo = "https://github.com/kubernetes/test-infra" + testImageRepo = prowRepo +) + +func Call(cmd string, args ...string) error { + c := exec.Command(cmd, args...) + c.Stdout = os.Stdout + c.Stderr = os.Stderr + return c.Run() +} + +// UpdatePR updates with github client "gc" the PR of github repo org/repo +// with "matchTitle" from "source" to "branch" +// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)" +// "images" and "extraLineInPRBody" are used to generate commit summary and body of the PR +func UpdatePR(gc github.Client, org, repo string, images map[string]string, extraLineInPRBody string, matchTitle, source, branch string) error { + return updatePR(gc, org, repo, makeCommitSummary(images), generatePRBody(images, extraLineInPRBody), matchTitle, source, branch) +} + +func updatePR(gc github.Client, org, repo, title, body, matchTitle, source, branch string) error { + logrus.Info("Creating PR...") + n, err := updater.UpdatePR(org, repo, title, body, matchTitle, gc) + if err != nil { + return fmt.Errorf("failed to update %d: %v", n, err) + } + if n == nil { + pr, err := gc.CreatePullRequest(org, repo, title, body, source, branch, true) + if err != nil { + return fmt.Errorf("failed to create PR: %v", err) + } + n = &pr + } + + logrus.Infof("PR %s/%s#%d will merge %s into %s: %s", org, repo, *n, source, branch, title) + return nil +} + +// UpdateReferences update the references of prow-images and testimages +// in the files in any of "subfolders" of the current dir +// if the file is a yaml file (*.yaml) or extraFiles[file]=true +func UpdateReferences(subfolders []string, extraFiles map[string]bool) (map[string]string, error) { + logrus.Info("Bumping image references...") + filter := regexp.MustCompile(prowPrefix + "|" + testImagePrefix) + + for _, dir := range subfolders { + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, ".yaml") || extraFiles[path] { + if err := bumper.UpdateFile(path, filter); err != nil { + logrus.WithError(err).Errorf("Failed to update path %s.", path) + } + } + return nil + }) + if err != nil { + return nil, err + } + } + + return bumper.GetReplacements(), nil +} + +func getNewProwVersion(images map[string]string) string { + for k, v := range images { + if strings.HasPrefix(k, prowPrefix) { + return v + } + } + return "" +} + +func makeCommitSummary(images map[string]string) string { + return fmt.Sprintf("Update prow to %s, and other images as necessary.", getNewProwVersion(images)) +} + +// MakeGitCommit runs a sequence of git commands to +// commit and push the changes the "remote" on "remoteBranch" +// "name" and "email" are used for git-commit command +// "images" contains the tag replacements that have been made which is returned from "UpdateReferences([]string{"."}, extraFiles)" +// "images" is used to generate commit message +func MakeGitCommit(remote, remoteBranch, name, email string, images map[string]string) error { + logrus.Info("Making git commit...") + if err := Call("git", "add", "-A"); err != nil { + return fmt.Errorf("failed to git add: %v", err) + } + message := makeCommitSummary(images) + commitArgs := []string{"commit", "-m", message} + if name != "" && email != "" { + commitArgs = append(commitArgs, "--author", fmt.Sprintf("%s <%s>", name, email)) + } + if err := Call("git", commitArgs...); err != nil { + return fmt.Errorf("failed to git commit: %v", err) + } + logrus.Info("Pushing to remote...") + if err := Call("git", "push", "-f", remote, fmt.Sprintf("HEAD:%s", remoteBranch)); err != nil { + return fmt.Errorf("failed to git push: %v", err) + } + return nil +} + +func tagFromName(name string) string { + parts := strings.Split(name, ":") + if len(parts) < 2 { + return "" + } + return parts[1] +} + +func componentFromName(name string) string { + s := strings.Split(strings.Split(name, ":")[0], "/") + return s[len(s)-1] +} + +func formatTagDate(d string) string { + if len(d) != 8 { + return d + } + // ‑ = U+2011 NON-BREAKING HYPHEN, to prevent line wraps. + return fmt.Sprintf("%s‑%s‑%s", d[0:4], d[4:6], d[6:8]) +} + +func generateSummary(name, repo, prefix string, summarise bool, images map[string]string) string { + type delta struct { + oldCommit string + newCommit string + oldDate string + newDate string + variant string + component string + } + versions := map[string][]delta{} + for image, newTag := range images { + if !strings.HasPrefix(image, prefix) { + continue + } + if strings.HasSuffix(image, ":"+newTag) { + continue + } + oldDate, oldCommit, oldVariant := bumper.DeconstructTag(tagFromName(image)) + newDate, newCommit, _ := bumper.DeconstructTag(newTag) + k := oldCommit + ":" + newCommit + d := delta{ + oldCommit: oldCommit, + newCommit: newCommit, + oldDate: oldDate, + newDate: newDate, + variant: oldVariant, + component: componentFromName(image), + } + versions[k] = append(versions[k], d) + } + + switch { + case len(versions) == 0: + return fmt.Sprintf("No %s changes.", name) + case len(versions) == 1 && summarise: + for k, v := range versions { + s := strings.Split(k, ":") + return fmt.Sprintf("%s changes: %s/compare/%s...%s (%s → %s)", name, repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate)) + } + default: + changes := make([]string, 0, len(versions)) + for k, v := range versions { + s := strings.Split(k, ":") + names := make([]string, 0, len(v)) + for _, d := range v { + names = append(names, d.component+d.variant) + } + sort.Strings(names) + changes = append(changes, fmt.Sprintf("%s/compare/%s...%s | %s → %s | %s", + repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate), strings.Join(names, ", "))) + } + sort.Slice(changes, func(i, j int) bool { return strings.Split(changes[i], "|")[1] < strings.Split(changes[j], "|")[1] }) + return fmt.Sprintf("Multiple distinct %s changes:\n\nCommits | Dates | Images\n--- | --- | ---\n%s\n", name, strings.Join(changes, "\n")) + } + panic("unreachable!") +} + +func generatePRBody(images map[string]string, assignment string) string { + prowSummary := generateSummary("Prow", prowRepo, prowPrefix, true, images) + testImagesSummary := generateSummary("test-image", testImageRepo, testImagePrefix, false, images) + return prowSummary + "\n\n" + testImagesSummary + "\n\n" + assignment + "\n" +} diff --git a/experiment/autobumper/main.go b/experiment/autobumper/main.go index 36cca6e9806c..32e2ae9759d7 100644 --- a/experiment/autobumper/main.go +++ b/experiment/autobumper/main.go @@ -23,27 +23,19 @@ import ( "net/http" "os" "os/exec" - "path/filepath" - "regexp" - "sort" "strings" "github.com/sirupsen/logrus" - "k8s.io/test-infra/experiment/image-bumper/bumper" + "k8s.io/test-infra/experiment/autobumper/bumper" "k8s.io/test-infra/prow/config/secret" "k8s.io/test-infra/prow/github" - "k8s.io/test-infra/robots/pr-creator/updater" ) const ( - prowPrefix = "gcr.io/k8s-prow/" - testImagePrefix = "gcr.io/k8s-testimages/" - prowRepo = "https://github.com/kubernetes/test-infra" - testImageRepo = prowRepo - oncallAddress = "https://storage.googleapis.com/kubernetes-jenkins/oncall.json" - githubOrg = "kubernetes" - githubRepo = "test-infra" + oncallAddress = "https://storage.googleapis.com/kubernetes-jenkins/oncall.json" + githubOrg = "kubernetes" + githubRepo = "test-infra" ) var extraFiles = map[string]bool{ @@ -97,166 +89,11 @@ func validateOptions(o options) error { return nil } -func call(cmd string, args ...string) error { - c := exec.Command(cmd, args...) - c.Stdout = os.Stdout - c.Stderr = os.Stderr - return c.Run() -} - -func updatePR(gc github.Client, org, repo, title, body, matchTitle, source, branch string) error { - logrus.Info("Creating PR...") - n, err := updater.UpdatePR(org, repo, title, body, matchTitle, gc) - if err != nil { - return fmt.Errorf("failed to update %d: %v", n, err) - } - if n == nil { - pr, err := gc.CreatePullRequest(org, repo, title, body, source, branch, true) - if err != nil { - return fmt.Errorf("failed to create PR: %v", err) - } - n = &pr - } - - logrus.Infof("PR %s/%s#%d will merge %s into %s: %s", org, repo, *n, source, branch, title) - return nil -} - -func updateReferences() (map[string]string, error) { - logrus.Info("Bumping image references...") - filter := regexp.MustCompile(prowPrefix + "|" + testImagePrefix) - - err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { - if strings.HasSuffix(path, ".yaml") || extraFiles[path] { - if err := bumper.UpdateFile(path, filter); err != nil { - logrus.WithError(err).Errorf("Failed to update path %s.", path) - } - } - return nil - }) - if err != nil { - return nil, err - } - - return bumper.GetReplacements(), nil -} - -func getNewProwVersion(images map[string]string) string { - for k, v := range images { - if strings.HasPrefix(k, prowPrefix) { - return v - } - } - return "" -} - -func makeCommitSummary(images map[string]string) string { - return fmt.Sprintf("Update prow to %s, and other images as necessary.", getNewProwVersion(images)) -} - func updateConfig() error { // Try to regenerate security job configs which use an explicit podutils image config // TODO(krzyzacy): workaround before we resolve https://github.com/kubernetes/test-infra/issues/9783 logrus.Info("Updating generated config...") - return call("bazel", "run", "//hack:update-config") -} - -func makeGitCommit(user, name, email string, images map[string]string) error { - logrus.Info("Making git commit...") - if err := call("git", "add", "-A"); err != nil { - return fmt.Errorf("failed to git add: %v", err) - } - message := makeCommitSummary(images) - commitArgs := []string{"commit", "-m", message} - if name != "" && email != "" { - commitArgs = append(commitArgs, "--author", fmt.Sprintf("%s <%s>", name, email)) - } - if err := call("git", commitArgs...); err != nil { - return fmt.Errorf("failed to git commit: %v", err) - } - logrus.Info("Pushing to remote...") - if err := call("git", "push", "-f", fmt.Sprintf("git@github.com:%s/test-infra.git", user), "HEAD:autobump"); err != nil { - return fmt.Errorf("failed to git push: %v", err) - } - return nil -} - -func tagFromName(name string) string { - parts := strings.Split(name, ":") - if len(parts) < 2 { - return "" - } - return parts[1] -} - -func componentFromName(name string) string { - s := strings.Split(strings.Split(name, ":")[0], "/") - return s[len(s)-1] -} - -func formatTagDate(d string) string { - if len(d) != 8 { - return d - } - // ‑ = U+2011 NON-BREAKING HYPHEN, to prevent line wraps. - return fmt.Sprintf("%s‑%s‑%s", d[0:4], d[4:6], d[6:8]) -} - -func generateSummary(name, repo, prefix string, summarise bool, images map[string]string) string { - type delta struct { - oldCommit string - newCommit string - oldDate string - newDate string - variant string - component string - } - versions := map[string][]delta{} - for image, newTag := range images { - if !strings.HasPrefix(image, prefix) { - continue - } - if strings.HasSuffix(image, ":"+newTag) { - continue - } - oldDate, oldCommit, oldVariant := bumper.DeconstructTag(tagFromName(image)) - newDate, newCommit, _ := bumper.DeconstructTag(newTag) - k := oldCommit + ":" + newCommit - d := delta{ - oldCommit: oldCommit, - newCommit: newCommit, - oldDate: oldDate, - newDate: newDate, - variant: oldVariant, - component: componentFromName(image), - } - versions[k] = append(versions[k], d) - } - - switch { - case len(versions) == 0: - return fmt.Sprintf("No %s changes.", name) - case len(versions) == 1 && summarise: - for k, v := range versions { - s := strings.Split(k, ":") - return fmt.Sprintf("%s changes: %s/compare/%s...%s (%s → %s)", name, repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate)) - } - default: - changes := make([]string, 0, len(versions)) - for k, v := range versions { - s := strings.Split(k, ":") - names := make([]string, 0, len(v)) - for _, d := range v { - names = append(names, d.component+d.variant) - } - sort.Strings(names) - changes = append(changes, fmt.Sprintf("%s/compare/%s...%s | %s → %s | %s", - repo, s[0], s[1], formatTagDate(v[0].oldDate), formatTagDate(v[0].newDate), strings.Join(names, ", "))) - } - sort.Slice(changes, func(i, j int) bool { return strings.Split(changes[i], "|")[1] < strings.Split(changes[j], "|")[1] }) - return fmt.Sprintf("Multiple distinct %s changes:\n\nCommits | Dates | Images\n--- | --- | ---\n%s\n", name, strings.Join(changes, "\n")) - } - panic("unreachable!") + return bumper.Call("bazel", "run", "//hack:update-config") } func getOncaller() (string, error) { @@ -279,23 +116,17 @@ func getOncaller() (string, error) { return oncall.Oncall.TestInfra, nil } -func generatePRBody(images map[string]string) string { - prowSummary := generateSummary("Prow", prowRepo, prowPrefix, true, images) - testImagesSummary := generateSummary("test-image", testImageRepo, testImagePrefix, false, images) +func getAssignment() string { oncaller, err := getOncaller() - - var assignment string if err == nil { if oncaller != "" { - assignment = "/cc @" + oncaller + return "/cc @" + oncaller } else { - assignment = "Nobody is currently oncall, so falling back to Blunderbuss." + return "Nobody is currently oncall, so falling back to Blunderbuss." } } else { - assignment = fmt.Sprintf("An error occurred while finding an assignee: `%s`.\nFalling back to Blunderbuss.", err) + return fmt.Sprintf("An error occurred while finding an assignee: `%s`.\nFalling back to Blunderbuss.", err) } - body := prowSummary + "\n\n" + testImagesSummary + "\n\n" + assignment + "\n" - return body } func main() { @@ -314,7 +145,7 @@ func main() { if err := cdToRootDir(); err != nil { logrus.WithError(err).Fatal("Failed to change to root dir") } - images, err := updateReferences() + images, err := bumper.UpdateReferences([]string{"."}, extraFiles) if err != nil { logrus.WithError(err).Fatal("Failed to update references.") } @@ -322,11 +153,12 @@ func main() { logrus.WithError(err).Fatal("Failed to update generated config.") } - if err := makeGitCommit(o.githubLogin, o.gitName, o.gitEmail, images); err != nil { + remoteBranch := "autobump" + if err := bumper.MakeGitCommit(fmt.Sprintf("git@github.com:%s/test-infra.git", o.githubLogin), remoteBranch, o.gitName, o.gitEmail, images); err != nil { logrus.WithError(err).Fatal("Failed to push changes.") } - if err := updatePR(gc, githubOrg, githubRepo, makeCommitSummary(images), generatePRBody(images), "Update prow to", o.githubLogin+":autobump", "master"); err != nil { + if err := bumper.UpdatePR(gc, githubOrg, githubRepo, images, getAssignment(), "Update prow to", o.githubLogin+":"+remoteBranch, "master"); err != nil { logrus.WithError(err).Fatal("PR creation failed.") } }