From f65c5228e9438312054029e3674ee2df4cff82a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E6=AD=A3=E6=B5=A9=2CZhu=20Zhenghao?= Date: Mon, 19 Jun 2023 23:07:25 +0800 Subject: [PATCH] add export logs command --- .github/workflows/test.yaml | 12 ++ pkg/kwokctl/cmd/export/export.go | 41 +++++++ pkg/kwokctl/cmd/export/logs/logs.go | 105 ++++++++++++++++ pkg/kwokctl/cmd/root.go | 2 + pkg/kwokctl/runtime/binary/cluster.go | 42 +++++++ pkg/kwokctl/runtime/compose/cluster.go | 60 +++++++++ pkg/kwokctl/runtime/config.go | 3 + pkg/kwokctl/runtime/kind/cluster.go | 64 ++++++++++ pkg/utils/exec/exec.go | 23 ++++ pkg/utils/file/file.go | 9 ++ site/content/en/docs/generated/kwokctl.md | 1 + .../en/docs/generated/kwokctl_export.md | 27 ++++ .../en/docs/generated/kwokctl_export_logs.md | 26 ++++ test/kwokctl/helper.sh | 13 +- .../kwokctl_binary_export_logs.test.sh | 31 +++++ .../kwokctl_docker_export_logs.test.sh | 31 +++++ test/kwokctl/kwokctl_exec_test.sh | 1 - test/kwokctl/kwokctl_export_logs_test.sh | 115 ++++++++++++++++++ .../kwokctl_kind-podman_export_logs.test.sh | 31 +++++ test/kwokctl/kwokctl_kind_export_logs.test.sh | 31 +++++ .../kwokctl_nerdctl_export_logs.test.sh | 31 +++++ .../kwokctl_podman_export_logs.test.sh | 31 +++++ test/kwokctl/kwokctl_scheduler_test.sh | 1 - test/kwokctl/kwokctl_workable_test.sh | 2 - test/kwokctl/suite.sh | 31 +---- 25 files changed, 733 insertions(+), 31 deletions(-) create mode 100644 pkg/kwokctl/cmd/export/export.go create mode 100644 pkg/kwokctl/cmd/export/logs/logs.go create mode 100644 site/content/en/docs/generated/kwokctl_export.md create mode 100644 site/content/en/docs/generated/kwokctl_export_logs.md create mode 100755 test/kwokctl/kwokctl_binary_export_logs.test.sh create mode 100755 test/kwokctl/kwokctl_docker_export_logs.test.sh create mode 100755 test/kwokctl/kwokctl_export_logs_test.sh create mode 100755 test/kwokctl/kwokctl_kind-podman_export_logs.test.sh create mode 100755 test/kwokctl/kwokctl_kind_export_logs.test.sh create mode 100755 test/kwokctl/kwokctl_nerdctl_export_logs.test.sh create mode 100755 test/kwokctl/kwokctl_podman_export_logs.test.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e28e9746c..1c479b87a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -311,6 +311,11 @@ jobs: run: | ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_logs + - name: Test Export Logs + shell: bash + run: | + ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_export_logs + - name: Test Attach shell: bash run: | @@ -346,3 +351,10 @@ jobs: KWOK_MODE: StableFeatureGateAndAPI run: | ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }} + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: failure() + with: + name: kwok-logs-${{ github.run_id }}-${{ matrix.os }}-${{ matrix.kwokctl-runtime }}${{ matrix.nerdctl-version && '-nerdctl' || '' }}${{ matrix.nerdctl-version }} + path: /tmp/kwok/logs diff --git a/pkg/kwokctl/cmd/export/export.go b/pkg/kwokctl/cmd/export/export.go new file mode 100644 index 000000000..7ead75892 --- /dev/null +++ b/pkg/kwokctl/cmd/export/export.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package export implements the `export` command +package export + +import ( + "context" + + "github.com/spf13/cobra" + + "sigs.k8s.io/kwok/pkg/kwokctl/cmd/export/logs" +) + +// NewCommand returns a new cobra.Command for export +func NewCommand(ctx context.Context) *cobra.Command { + cmd := &cobra.Command{ + Args: cobra.NoArgs, + Use: "export", + Short: "Exports one of [logs]", + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + // add subcommands + cmd.AddCommand(logs.NewCommand(ctx)) + return cmd +} diff --git a/pkg/kwokctl/cmd/export/logs/logs.go b/pkg/kwokctl/cmd/export/logs/logs.go new file mode 100644 index 000000000..a2facfa7e --- /dev/null +++ b/pkg/kwokctl/cmd/export/logs/logs.go @@ -0,0 +1,105 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package logs implements the `logs` command +package logs + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + + "github.com/spf13/cobra" + + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/file" +) + +type flagpole struct { + Name string +} + +// NewCommand returns a new cobra.Command for getting the cluster logs +func NewCommand(ctx context.Context) *cobra.Command { + flags := &flagpole{} + cmd := &cobra.Command{ + Args: cobra.MaximumNArgs(1), + Use: "logs [output-dir]", + Short: "Exports logs to a tempdir or [output-dir] if specified", + RunE: func(cmd *cobra.Command, args []string) error { + flags.Name = config.DefaultCluster + return runE(ctx, flags, args) + }, + } + return cmd +} + +func runE(ctx context.Context, flags *flagpole, args []string) error { + name := config.ClusterName(flags.Name) + workdir := path.Join(config.ClustersDir, flags.Name) + + logger := log.FromContext(ctx).With("cluster", flags.Name) + ctx = log.NewContext(ctx, logger) + + rt, err := runtime.DefaultRegistry.Load(ctx, name, workdir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + logger.Warn("Cluster is not exists") + } + return err + } + + // get the optional directory argument, or create a tempdir under the kwok default working directory + var dir string + if len(args) == 0 || args[0] == "" { + tmp := filepath.Join(workdir, "tmp") + if err := os.MkdirAll(tmp, 0750); err != nil { + return fmt.Errorf("failed to create tmp directory: %w", err) + } + t, err := os.MkdirTemp(tmp, "log-") + if err != nil { + return err + } + dir = t + } else { + dir = filepath.Join(args[0], name) + } + if err := os.MkdirAll(dir, 0750); err != nil { + return fmt.Errorf("failed to create logs directory: %w", err) + } + + kwokConfigPath := filepath.Join(dir, "kwok.yaml") + if _, err := os.Stat(kwokConfigPath); err == nil { + return fmt.Errorf("%s already exists", kwokConfigPath) + } + logger.Info("Exporting logs", "dir", dir) + + err = file.Copy(rt.GetWorkdirPath(runtime.ConfigName), kwokConfigPath) + if err != nil { + return err + } + + if err = rt.CollectLogs(ctx, name, dir); err != nil { + return err + } + + return nil +} diff --git a/pkg/kwokctl/cmd/root.go b/pkg/kwokctl/cmd/root.go index 7e1b4ed38..f584a1420 100644 --- a/pkg/kwokctl/cmd/root.go +++ b/pkg/kwokctl/cmd/root.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/kwok/pkg/kwokctl/cmd/create" del "sigs.k8s.io/kwok/pkg/kwokctl/cmd/delete" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/etcdctl" + "sigs.k8s.io/kwok/pkg/kwokctl/cmd/export" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/get" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/kubectl" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/logs" @@ -62,6 +63,7 @@ func NewCommand(ctx context.Context) *cobra.Command { etcdctl.NewCommand(ctx), logs.NewCommand(ctx), snapshot.NewCommand(ctx), + export.NewCommand(ctx), ) return cmd } diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 4374fed20..47e451028 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -22,12 +22,14 @@ import ( "io" "os" "path/filepath" + rt "runtime" "time" "github.com/nxadm/tail" "golang.org/x/sync/errgroup" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/components" "sigs.k8s.io/kwok/pkg/kwokctl/k8s" "sigs.k8s.io/kwok/pkg/kwokctl/pki" @@ -762,6 +764,46 @@ func (c *Cluster) LogsFollow(ctx context.Context, name string, out io.Writer) er return nil } +// CollectLogs returns the logs of the specified component. +func (c *Cluster) CollectLogs(ctx context.Context, name string, dir string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + + path := filepath.Join(dir, consts.RuntimeTypeBinary+"-info.txt") + f, err := file.Open(path, 0640) + if err != nil { + return err + } + _, err = f.WriteString(fmt.Sprintf("%s/%s", rt.GOOS, rt.GOARCH)) + if err != nil { + return err + } + if err = f.Close(); err != nil { + return err + } + + componentsDir := filepath.Join(dir, "components") + logger := log.FromContext(ctx) + for _, component := range conf.Components { + src := c.GetLogPath(filepath.Base(component.Name) + ".log") + dest := filepath.Join(componentsDir, component.Name+".log") + if err = file.Copy(src, dest); err != nil { + logger.Error("Failed to copy file", err) + } + } + if conf.Options.KubeAuditPolicy != "" { + src := c.GetLogPath(runtime.AuditLogName) + dest := filepath.Join(componentsDir, runtime.AuditLogName) + if err = file.Copy(src, dest); err != nil { + logger.Error("Failed to copy file", err) + } + } + + return nil +} + // ListBinaries list binaries in the cluster func (c *Cluster) ListBinaries(ctx context.Context) ([]string, error) { config, err := c.Config(ctx) diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 98622568e..5fca1122e 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strconv" "strings" @@ -766,6 +767,65 @@ func (c *Cluster) LogsFollow(ctx context.Context, name string, out io.Writer) er return c.logs(ctx, name, out, true) } +// CollectLogs returns the logs of the specified component. +func (c *Cluster) CollectLogs(ctx context.Context, name string, dir string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + + err = exec.WriteToPath(ctx, filepath.Join(dir, c.runtime+"-info.txt"), []string{c.runtime, "info"}) + if err != nil { + return err + } + + componentsDir := filepath.Join(dir, "components") + logger := log.FromContext(ctx) + for _, component := range conf.Components { + path := filepath.Join(componentsDir, component.Name+".log") + f, err := file.Open(path, 0640) + if err != nil { + logger.Error("Failed to open file", err) + continue + } + if err = c.Logs(ctx, component.Name, f); err != nil { + logger.Error("Failed to get log", err) + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(path); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(path); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + + if conf.Options.KubeAuditPolicy != "" { + filePath := filepath.Join(componentsDir, "audit.log") + f, err := file.Open(filePath, 0640) + if err != nil { + logger.Error("Failed to open file", err) + } else { + if err = c.AuditLogs(ctx, f); err != nil { + logger.Error("Failed to get audit log", err) + } + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(filePath); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + } + + return nil +} + // ListBinaries list binaries in the cluster func (c *Cluster) ListBinaries(ctx context.Context) ([]string, error) { config, err := c.Config(ctx) diff --git a/pkg/kwokctl/runtime/config.go b/pkg/kwokctl/runtime/config.go index e1b14e239..876783a1b 100644 --- a/pkg/kwokctl/runtime/config.go +++ b/pkg/kwokctl/runtime/config.go @@ -92,6 +92,9 @@ type Runtime interface { // LogsFollow follow logs of a component with follow LogsFollow(ctx context.Context, name string, out io.Writer) error + // CollectLogs will populate dir with cluster logs and other debug files + CollectLogs(ctx context.Context, name string, dir string) error + // AuditLogs audit logs of apiserver AuditLogs(ctx context.Context, out io.Writer) error diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 947b9f8e9..c42a24b37 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -676,6 +676,70 @@ func (c *Cluster) LogsFollow(ctx context.Context, name string, out io.Writer) er return c.logs(ctx, name, out, true) } +// CollectLogs returns the logs of the specified component. +func (c *Cluster) CollectLogs(ctx context.Context, name string, dir string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + + kindPath, err := c.preDownloadKind(ctx) + if err != nil { + return err + } + ctx = c.withProviderEnv(ctx) + err = exec.WriteToPath(ctx, filepath.Join(dir, conf.Options.Runtime+"-info.txt"), []string{kindPath, "version"}) + if err != nil { + return err + } + + componentsDir := filepath.Join(dir, "components") + logger := log.FromContext(ctx) + for _, component := range conf.Components { + path := filepath.Join(componentsDir, component.Name+".log") + f, err := file.Open(path, 0640) + if err != nil { + logger.Error("Failed to open file", err) + continue + } + if err = c.Logs(ctx, component.Name, f); err != nil { + logger.Error("Failed to get log", err) + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(path); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(path); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + + if conf.Options.KubeAuditPolicy != "" { + filePath := filepath.Join(componentsDir, "audit.log") + f, err := file.Open(filePath, 0640) + if err != nil { + logger.Error("Failed to open file", err) + } else { + if err = c.AuditLogs(ctx, f); err != nil { + logger.Error("Failed to get audit log", err) + } + if err = f.Close(); err != nil { + logger.Error("Failed to close file", err) + if err = os.Remove(filePath); err != nil { + logger.Error("Failed to remove file", err) + } + } + } + } + + return nil +} + // ListBinaries list binaries in the cluster func (c *Cluster) ListBinaries(ctx context.Context) ([]string, error) { config, err := c.Config(ctx) diff --git a/pkg/utils/exec/exec.go b/pkg/utils/exec/exec.go index e13b523af..b2816e710 100644 --- a/pkg/utils/exec/exec.go +++ b/pkg/utils/exec/exec.go @@ -23,6 +23,9 @@ import ( "io" "os" "strings" + + "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/file" ) // IOStreams contains the standard streams. @@ -175,3 +178,23 @@ func Exec(ctx context.Context, name string, arg ...string) error { } return nil } + +// WriteToPath writes the output of a command to a specified file +func WriteToPath(ctx context.Context, path string, commands []string) error { + f, err := file.Open(path, 0640) + if err != nil { + return err + } + defer func() { + err = f.Close() + if err != nil { + logger := log.FromContext(ctx) + logger.Error("Failed to close file", err) + err := os.Remove(path) + if err != nil { + logger.Error("Failed to remove file", err) + } + } + }() + return Exec(WithAllWriteTo(ctx, f), commands[0], commands[1:]...) +} diff --git a/pkg/utils/file/file.go b/pkg/utils/file/file.go index 9075873a1..b1865fe18 100644 --- a/pkg/utils/file/file.go +++ b/pkg/utils/file/file.go @@ -101,3 +101,12 @@ func Exists(name string) bool { func Remove(name string) error { return os.Remove(name) } + +// Open open or create the file +func Open(name string, perm os.FileMode) (*os.File, error) { + err := os.MkdirAll(filepath.Dir(name), 0750) + if err != nil { + return nil, err + } + return os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) +} diff --git a/site/content/en/docs/generated/kwokctl.md b/site/content/en/docs/generated/kwokctl.md index e70042426..3012a9339 100644 --- a/site/content/en/docs/generated/kwokctl.md +++ b/site/content/en/docs/generated/kwokctl.md @@ -20,6 +20,7 @@ kwokctl [command] [flags] * [kwokctl create](kwokctl_create.md) - Creates one of [cluster] * [kwokctl delete](kwokctl_delete.md) - Deletes one of [cluster] * [kwokctl etcdctl](kwokctl_etcdctl.md) - etcdctl in cluster +* [kwokctl export](kwokctl_export.md) - Exports one of [logs] * [kwokctl get](kwokctl_get.md) - Gets one of [artifacts, clusters, kubeconfig] * [kwokctl kubectl](kwokctl_kubectl.md) - kubectl in cluster * [kwokctl logs](kwokctl_logs.md) - Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, prometheus] diff --git a/site/content/en/docs/generated/kwokctl_export.md b/site/content/en/docs/generated/kwokctl_export.md new file mode 100644 index 000000000..bf106d2d5 --- /dev/null +++ b/site/content/en/docs/generated/kwokctl_export.md @@ -0,0 +1,27 @@ +## kwokctl export + +Exports one of [logs] + +``` +kwokctl export [flags] +``` + +### Options + +``` + -h, --help help for export +``` + +### Options inherited from parent commands + +``` + -c, --config stringArray config path (default [~/.kwok/kwok.yaml]) + --name string cluster name (default "kwok") + -v, --v log-level number for the log level verbosity (DEBUG, INFO, WARN, ERROR) or (-4, 0, 4, 8) (default INFO) +``` + +### SEE ALSO + +* [kwokctl](kwokctl.md) - kwokctl is a tool to streamline the creation and management of clusters, with nodes simulated by kwok +* [kwokctl export logs](kwokctl_export_logs.md) - Exports logs to a tempdir or [output-dir] if specified + diff --git a/site/content/en/docs/generated/kwokctl_export_logs.md b/site/content/en/docs/generated/kwokctl_export_logs.md new file mode 100644 index 000000000..de70b73bb --- /dev/null +++ b/site/content/en/docs/generated/kwokctl_export_logs.md @@ -0,0 +1,26 @@ +## kwokctl export logs + +Exports logs to a tempdir or [output-dir] if specified + +``` +kwokctl export logs [output-dir] [flags] +``` + +### Options + +``` + -h, --help help for logs +``` + +### Options inherited from parent commands + +``` + -c, --config stringArray config path (default [~/.kwok/kwok.yaml]) + --name string cluster name (default "kwok") + -v, --v log-level number for the log level verbosity (DEBUG, INFO, WARN, ERROR) or (-4, 0, 4, 8) (default INFO) +``` + +### SEE ALSO + +* [kwokctl export](kwokctl_export.md) - Exports one of [logs] + diff --git a/test/kwokctl/helper.sh b/test/kwokctl/helper.sh index f2fd4dfb1..e6c045527 100644 --- a/test/kwokctl/helper.sh +++ b/test/kwokctl/helper.sh @@ -20,6 +20,7 @@ DIR="$(realpath "${DIR}")" ROOT_DIR="$(realpath "${DIR}/../..")" source "${ROOT_DIR}/hack/requirements.sh" +source "${DIR}/suite.sh" VERSION="$("${ROOT_DIR}/hack/get-version.sh")" @@ -39,7 +40,17 @@ function test_all() { local releases=("${@:3}") echo "Test ${cases} on ${runtime} for ${releases[*]}" - KWOK_RUNTIME="${runtime}" "${DIR}/kwokctl_${cases}_test.sh" "${releases[@]}" + if KWOK_RUNTIME="${runtime}" "${DIR}/kwokctl_${cases}_test.sh" "${releases[@]}"; then + rm -rf "${KWOK_LOGS_DIR}" + else + return 1 + fi + + for name in $(kwokctl get clusters); do + echo "Clean up cluster '${name}' that have not been deleted." + delete_cluster "${name}" + return 1 + done } # Test only the latest releases of Kubernetes diff --git a/test/kwokctl/kwokctl_binary_export_logs.test.sh b/test/kwokctl/kwokctl_binary_export_logs.test.sh new file mode 100755 index 000000000..f87e5ef40 --- /dev/null +++ b/test/kwokctl/kwokctl_binary_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "binary" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements_for_binary + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_docker_export_logs.test.sh b/test/kwokctl/kwokctl_docker_export_logs.test.sh new file mode 100755 index 000000000..2b64aee32 --- /dev/null +++ b/test/kwokctl/kwokctl_docker_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "docker" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_exec_test.sh b/test/kwokctl/kwokctl_exec_test.sh index ecd32d280..a2f432a4c 100755 --- a/test/kwokctl/kwokctl_exec_test.sh +++ b/test/kwokctl/kwokctl_exec_test.sh @@ -56,7 +56,6 @@ function test_exec() { echo "Error: exec result does not match" echo " want: ${want}" echo " got: ${result}" - show_info "${name}" return 1 fi } diff --git a/test/kwokctl/kwokctl_export_logs_test.sh b/test/kwokctl/kwokctl_export_logs_test.sh new file mode 100755 index 000000000..978bc6757 --- /dev/null +++ b/test/kwokctl/kwokctl_export_logs_test.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/suite.sh" + +RELEASES=() + +function usage() { + echo "Usage: $0 " + echo " is the version of kubernetes to test against." +} + +function args() { + if [[ $# -eq 0 ]]; then + usage + exit 1 + fi + while [[ $# -gt 0 ]]; do + RELEASES+=("${1}") + shift + done +} + +function test_export_logs() { + local name="${1}" + local root_dir="${KWOK_LOGS_DIR}/kwok-${name}" + + if ! save_logs "${name}"; then + echo "Error: export logs failed" + return 1 + fi + + tree "${KWOK_LOGS_DIR}" + + if [ ! -d "${root_dir}" ]; then + echo "Required directory ${root_dir} does not exist." + return 1 + fi + + REQUIRED_FILES=( + "kwok.yaml" + "${KWOK_RUNTIME}-info.txt" + ) + + for file in "${REQUIRED_FILES[@]}"; do + if [ ! -f "${root_dir}/$file" ]; then + echo "Required file $root_dir/$file does not exist." + return 1 + fi + done + + if [ ! -d "${root_dir}/components" ]; then + echo "Required directory ${root_dir}/components does not exist." + return 1 + fi + + LOG_FILES=( + "audit.log" + "etcd.log" + "kube-apiserver.log" + "kube-controller-manager.log" + "kube-scheduler.log" + "kwok-controller.log" + ) + for file in "${LOG_FILES[@]}"; do + if [ ! -f "${root_dir}/components/$file" ]; then + echo "Required file ${root_dir}/components/$file does not exist." + return 1 + fi + done + + echo "Directory $KWOK_LOGS_DIR is correct." +} + +function main() { + local failed=() + + for release in "${RELEASES[@]}"; do + echo "------------------------------" + echo "Testing export logs on ${KWOK_RUNTIME} for ${release}" + name="export-logs-cluster-${KWOK_RUNTIME}-${release//./-}" + create_cluster "${name}" "${release}" --kube-audit-policy="${DIR}/audit-policy.yaml" -v=4 + test_export_logs "${name}" || failed+=("${name}_export_logs") + delete_cluster "${name}" + done + + if [[ "${#failed[@]}" -ne 0 ]]; then + echo "------------------------------" + echo "Error: Some tests failed" + for test in "${failed[@]}"; do + echo " - ${test}" + done + exit 1 + fi +} + +args "$@" + +main diff --git a/test/kwokctl/kwokctl_kind-podman_export_logs.test.sh b/test/kwokctl/kwokctl_kind-podman_export_logs.test.sh new file mode 100755 index 000000000..3f1214156 --- /dev/null +++ b/test/kwokctl/kwokctl_kind-podman_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "kind-podman" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements_for_podman + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_kind_export_logs.test.sh b/test/kwokctl/kwokctl_kind_export_logs.test.sh new file mode 100755 index 000000000..b262ab851 --- /dev/null +++ b/test/kwokctl/kwokctl_kind_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "kind" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_nerdctl_export_logs.test.sh b/test/kwokctl/kwokctl_nerdctl_export_logs.test.sh new file mode 100755 index 000000000..3ce5b9665 --- /dev/null +++ b/test/kwokctl/kwokctl_nerdctl_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "nerdctl" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements_for_nerdctl + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_podman_export_logs.test.sh b/test/kwokctl/kwokctl_podman_export_logs.test.sh new file mode 100755 index 000000000..da99c5a16 --- /dev/null +++ b/test/kwokctl/kwokctl_podman_export_logs.test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +source "${DIR}/helper.sh" + +function main() { + local all_releases=("${@}") + + test_all "podman" "export_logs" "${all_releases[@]}" || exit 1 +} + +requirements_for_podman + +mapfile -t releases < <(supported_releases) +main "${releases[@]}" diff --git a/test/kwokctl/kwokctl_scheduler_test.sh b/test/kwokctl/kwokctl_scheduler_test.sh index 22fe2ccb8..de78f227c 100755 --- a/test/kwokctl/kwokctl_scheduler_test.sh +++ b/test/kwokctl/kwokctl_scheduler_test.sh @@ -58,7 +58,6 @@ function test_scheduler() { if ! kwokctl --name="${name}" kubectl get pod | grep Running >/dev/null 2>&1; then echo "Error: cluster not ready" - show_all return 1 fi } diff --git a/test/kwokctl/kwokctl_workable_test.sh b/test/kwokctl/kwokctl_workable_test.sh index f81c6041a..9f571e391 100755 --- a/test/kwokctl/kwokctl_workable_test.sh +++ b/test/kwokctl/kwokctl_workable_test.sh @@ -64,7 +64,6 @@ function test_workable() { if ! kwokctl --name="${name}" kubectl get pod | grep Running >/dev/null 2>&1; then echo "Error: cluster not ready" - show_all return 1 fi @@ -78,7 +77,6 @@ function test_workable() { if ! kwokctl --name="${name}" etcdctl get /registry/namespaces/default --keys-only | grep default >/dev/null 2>&1; then echo "Error: Failed to get namespace(default) by kwokctl etcdctl in cluster ${name}" - show_all return 1 fi } diff --git a/test/kwokctl/suite.sh b/test/kwokctl/suite.sh index 5beb846fe..c4cf104ac 100644 --- a/test/kwokctl/suite.sh +++ b/test/kwokctl/suite.sh @@ -13,33 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -function show_all() { - for name in $(kwokctl get clusters); do - show_info "${name}" - done -} +export KWOK_LOGS_DIR="/tmp/kwok/logs" -function show_info() { +function save_logs() { local name="${1}" - echo - echo kwokctl --name="${name}" kubectl get pod -o wide --all-namespaces - kwokctl --name="${name}" kubectl get pod -o wide --all-namespaces - echo - echo kwokctl --name="${name}" logs etcd - kwokctl --name="${name}" logs etcd - echo - echo kwokctl --name="${name}" logs kube-apiserver - kwokctl --name="${name}" logs kube-apiserver - echo - echo kwokctl --name="${name}" logs kube-controller-manager - kwokctl --name="${name}" logs kube-controller-manager - echo - echo kwokctl --name="${name}" logs kube-scheduler - kwokctl --name="${name}" logs kube-scheduler - echo - echo kwokctl --name="${name}" logs kwok-controller - kwokctl --name="${name}" logs kwok-controller - echo + mkdir -p "${KWOK_LOGS_DIR}" + kwokctl --name="${name}" export logs "${KWOK_LOGS_DIR}" } function create_cluster() { @@ -56,13 +35,13 @@ function create_cluster() { --disable-qps-limits \ "$@"; then echo "Error: Cluster ${name} creation failed" - show_all exit 1 fi } function delete_cluster() { local name="${1}" + save_logs "${name}" if ! kwokctl delete cluster --name "${name}"; then echo "Error: Cluster ${name} deletion failed" exit 1