From d76512cf9230827fbea55fca2087f599308b7300 Mon Sep 17 00:00:00 2001 From: Javier Date: Tue, 12 Sep 2023 09:19:59 -0600 Subject: [PATCH] commands: add command to wiping cluster Added command destroy-cluster for remove all crds Added documentation for destroy-cluster Added unittest and integration tests Signed-off-by: Javier --- .github/workflows/go-test.yaml | 17 +++ Makefile | 5 + README.md | 44 ++++++ cmd/commands/destroy_cluster.go | 63 +++++++++ cmd/commands/root.go | 6 + cmd/main.go | 1 + docs/destroy_cluster.md | 108 ++++++++++++++ docs/health.md | 2 +- go.mod | 1 + go.sum | 1 + pkg/crds/crds.go | 160 +++++++++++++++++++++ pkg/crds/crds_test.go | 243 ++++++++++++++++++++++++++++++++ pkg/k8sutil/context.go | 4 + pkg/k8sutil/dynamic.go | 121 ++++++++++++++++ pkg/k8sutil/interface.go | 31 ++++ pkg/k8sutil/mocks.go | 95 +++++++++++++ 16 files changed, 901 insertions(+), 1 deletion(-) create mode 100644 cmd/commands/destroy_cluster.go create mode 100644 docs/destroy_cluster.md create mode 100644 pkg/crds/crds.go create mode 100644 pkg/crds/crds_test.go create mode 100644 pkg/k8sutil/dynamic.go create mode 100644 pkg/k8sutil/interface.go create mode 100644 pkg/k8sutil/mocks.go diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index bf88599f..c774fb3c 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -225,6 +225,23 @@ jobs: kubectl -n test-cluster scale deployment rook-ceph-osd-0 --replicas 0 kubectl rook-ceph --operator-namespace test-operator -n test-cluster rook purge-osd 0 --force + - name: List CRDS + run: | + set -ex + kubectl -n test-cluster get all + + - name: Destroy Cluster (removing CRDS) + env: + ROOK_PLUGIN_SKIP_PROMPTS: true + run: | + set -ex + kubectl rook-ceph --operator-namespace test-operator -n test-cluster destroy-cluster + + - name: Validate destroyed CRDS + run: | + set -ex + kubectl get deployments -n rook-ceph --no-headers| wc -l | (read n && [ $n -le 1 ] || { echo "the crds could not be deleted"; exit 1;}) + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs diff --git a/Makefile b/Makefile index 9a4b03f7..60ebd9eb 100644 --- a/Makefile +++ b/Makefile @@ -29,3 +29,8 @@ help : @echo "build : Create go binary." @echo "test : Runs unit tests" @echo "clean : Remove go binary file." + +generate: + @echo "generating mocks..." + @go generate ./... + @echo completed \ No newline at end of file diff --git a/README.md b/README.md index 8bd0e89a..e774bf88 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Visit docs below for complete details about each command and their flags uses. 1. [Debug OSDs and Mons](docs/debug.md) 1. [Restore mon quorum](docs/mons.md#restore-quorum) 1. [Disaster Recovery](docs/dr-health.md) +1. [Destroy Cluster](docs/destroy_cluster.md) ## Examples @@ -178,6 +179,49 @@ kubectl rook-ceph ceph versions } } ``` +### Destroy Cluster +```bash +$ kubectl rook-ceph -n rook-ceph destroy-cluster +Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"? +yes-really-destroy-cluster +Info: proceeding +Info: Getting resources kind cephblockpoolradosnamespaces +Warning: resource cephblockpoolradosnamespaces was not found on the cluster +Info: Getting resources kind cephblockpools +Warning: resource cephblockpools was not found on the cluster +Info: Getting resources kind cephbucketnotifications +Warning: resource cephbucketnotifications was not found on the cluster +Info: Getting resources kind cephbuckettopics +Warning: resource cephbuckettopics was not found on the cluster +Info: Getting resources kind cephclients +Warning: resource cephclients was not found on the cluster +Info: Getting resources kind cephclusters +Warning: resource cephclusters was not found on the cluster +Info: Getting resources kind cephcosidrivers +Warning: resource cephcosidrivers was not found on the cluster +Info: Getting resources kind cephfilesystemmirrors +Warning: resource cephfilesystemmirrors was not found on the cluster +Info: Getting resources kind cephfilesystems +Warning: resource cephfilesystems was not found on the cluster +Info: Getting resources kind cephfilesystemsubvolumegroup +Warning: the server could not find the requested resource: cephfilesystemsubvolumegroup +Info: Getting resources kind cephnfses +Warning: resource cephnfses was not found on the cluster +Info: Getting resources kind cephobjectrealms +Warning: resource cephobjectrealms was not found on the cluster +Info: Getting resources kind cephobjectstores +Warning: resource cephobjectstores was not found on the cluster +Info: Getting resources kind cephobjectstoreusers +Warning: resource cephobjectstoreusers was not found on the cluster +Info: Getting resources kind cephobjectzonegroups +Warning: resource cephobjectzonegroups was not found on the cluster +Info: Getting resources kind cephobjectzones +Warning: resource cephobjectzones was not found on the cluster +Info: Getting resources kind cephrbdmirrors +Warning: resource cephrbdmirrors was not found on the cluster +Info: done +$ +``` ## Contributing diff --git a/cmd/commands/destroy_cluster.go b/cmd/commands/destroy_cluster.go new file mode 100644 index 00000000..8cef975b --- /dev/null +++ b/cmd/commands/destroy_cluster.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + "github.com/rook/kubectl-rook-ceph/pkg/crds" + "github.com/rook/kubectl-rook-ceph/pkg/logging" + "github.com/spf13/cobra" + "os" + "strings" +) + +const ( + destroyClusterQuestion = "Are you sure you want to destroy the cluster in namespace %q? If absolutely certain, enter: " + destroyClusterAnswer + destroyClusterAnswer = "yes-really-destroy-cluster" +) + +// DestroyClusterCmd represents the command for destroy cluster +var DestroyClusterCmd = &cobra.Command{ + Use: "destroy-cluster", + Short: "delete all Rook CRDs", + Run: func(cmd *cobra.Command, args []string) { + question := fmt.Sprintf(destroyClusterQuestion, CephClusterNamespace) + confirmation, err := confirmPrompt(question, destroyClusterAnswer) + if err != nil { + logging.Fatal(err) + } + + logging.Info(confirmation) + clientsets := GetClientsets(cmd.Context()) + crds.DeleteCustomResources(cmd.Context(), clientsets, CephClusterNamespace) + }, +} + +func confirmPrompt(question, key string) (string, error) { + var answer string + if skip, ok := os.LookupEnv("ROOK_PLUGIN_SKIP_PROMPTS"); ok && skip == "true" { + return "skipped prompt since ROOK_PLUGIN_SKIP_PROMPTS=true", nil + } + logging.Warning(question) + fmt.Scan(&answer) + + if strings.EqualFold(answer, key) { + return "proceeding", nil + } + + return "", fmt.Errorf("the response %q to confirm the cluster deletion", destroyClusterAnswer) +} diff --git a/cmd/commands/root.go b/cmd/commands/root.go index ede154d1..376353da 100644 --- a/cmd/commands/root.go +++ b/cmd/commands/root.go @@ -18,6 +18,7 @@ package command import ( "context" "fmt" + "k8s.io/client-go/dynamic" "regexp" "strings" @@ -102,6 +103,11 @@ func GetClientsets(ctx context.Context) *k8sutil.Clientsets { logging.Fatal(err) } + clientsets.Dynamic, err = dynamic.NewForConfig(clientsets.KubeConfig) + if err != nil { + logging.Fatal(err) + } + PreValidationCheck(ctx, clientsets, OperatorNamespace, CephClusterNamespace) return clientsets diff --git a/cmd/main.go b/cmd/main.go index 0d43a42d..7b4bdcda 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,5 +38,6 @@ func addcommands() { command.DebugCmd, command.Health, command.DrCmd, + command.DestroyClusterCmd, ) } diff --git a/docs/destroy_cluster.md b/docs/destroy_cluster.md new file mode 100644 index 00000000..fdc45a74 --- /dev/null +++ b/docs/destroy_cluster.md @@ -0,0 +1,108 @@ +# Destroy Cluster + +This command is used for destroy CRDS (custom resource definitions) created by rook-ceph + +## !!! Warning !!! + **This command is not reversible**, and it will destroy your rook-ceph cluster completely and your data, + please only use this command if your sure that your data must be destroyed. + +## How to use: + +Just you need to run the command, and you will be asked for the confirmation +```bash +$ kubectl rook-ceph -n rook-ceph destroy-cluster +Are you sure you want to destroy the cluster in namespace "rook-ceph"? | +``` + +You must write exactly this answer **yes-really-destroy-cluster** +to confirm that you really want to execute the command and destroy +your cluster, any other response will be rejected and +the process will be stopped even responding yes + +```bash +$ kubectl rook-ceph -n rook-ceph destroy-cluster +Are you sure you want to destroy the cluster in namespace "rook-ceph"? yes +Error: you need to response "yes-really-destroy-cluster" to confirm the cluster deletion +exit status 1 + +``` + +Once you respond with the right answer the process will start, and it will be deleting the CRDS created by rook-ceph + +```bash +$ kubectl rook-ceph -n rook-ceph destroy-cluster +Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"? +yes-really-destroy-cluster +Info: proceeding +Info: Getting resources kind cephblockpoolradosnamespaces +Warning: resource cephblockpoolradosnamespaces was not found on the cluster +Info: Getting resources kind cephblockpools +Warning: resource cephblockpools was not found on the cluster +Info: Getting resources kind cephbucketnotifications +Warning: resource cephbucketnotifications was not found on the cluster +Info: Getting resources kind cephbuckettopics +Warning: resource cephbuckettopics was not found on the cluster +Info: Getting resources kind cephclients +Warning: resource cephclients was not found on the cluster +Info: Getting resources kind cephclusters +Warning: resource cephclusters was not found on the cluster +Info: Getting resources kind cephcosidrivers +Warning: resource cephcosidrivers was not found on the cluster +Info: Getting resources kind cephfilesystemmirrors +Warning: resource cephfilesystemmirrors was not found on the cluster +Info: Getting resources kind cephfilesystems +Warning: resource cephfilesystems was not found on the cluster +Info: Getting resources kind cephfilesystemsubvolumegroup +Warning: the server could not find the requested resource: cephfilesystemsubvolumegroup +Info: Getting resources kind cephnfses +Warning: resource cephnfses was not found on the cluster +Info: Getting resources kind cephobjectrealms +Warning: resource cephobjectrealms was not found on the cluster +Info: Getting resources kind cephobjectstores +Warning: resource cephobjectstores was not found on the cluster +Info: Getting resources kind cephobjectstoreusers +Warning: resource cephobjectstoreusers was not found on the cluster +Info: Getting resources kind cephobjectzonegroups +Warning: resource cephobjectzonegroups was not found on the cluster +Info: Getting resources kind cephobjectzones +Warning: resource cephobjectzones was not found on the cluster +Info: Getting resources kind cephrbdmirrors +Warning: resource cephrbdmirrors was not found on the cluster +Info: done +$ +``` + + +# Developer changes +We are adding unittests in order to ensure that the destroy-cluster command is working as we expected, +so if you want to expand the functionality, and you need to change the `pgk/k8sutil/interface.go` remember to recreate the mocks by using `make generate` + +```bash +$ make generate +generating mocks... +completed +$ +``` + +# Run test + +To run the tests you should to execute `make test` +```bash +$ make test +running unit tests +go test ./... +? github.com/rook/kubectl-rook-ceph/cmd [no test files] +ok github.com/rook/kubectl-rook-ceph/cmd/commands (cached) +ok github.com/rook/kubectl-rook-ceph/pkg/crds (cached) +? github.com/rook/kubectl-rook-ceph/pkg/debug [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/dr [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/exec [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/health [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/k8sutil [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/logging [no test files] +? github.com/rook/kubectl-rook-ceph/pkg/rook [no test files] +ok github.com/rook/kubectl-rook-ceph/pkg/mons (cached) +$ +``` + + diff --git a/docs/health.md b/docs/health.md index c6a1ae29..3219c4a8 100644 --- a/docs/health.md +++ b/docs/health.md @@ -13,7 +13,7 @@ Health commands logs have three ways of logging: 1. `Info`: This is just a logging information for the users. 2. `Warning`: which mean there is some improvement required in the cluster. -3. `Error`: This reuires immediate user attentions to get the cluster in healthy state. +3. `Error`: This requires immediate user attentions to get the cluster in healthy state. ## Output diff --git a/go.mod b/go.mod index bceb7239..7f953ebc 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/fatih/color v1.15.0 + github.com/golang/mock v1.6.0 github.com/rook/rook v1.12.2 github.com/rook/rook/pkg/apis v0.0.0-20230725213142-5979b3816292 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index 7239323d..af5865ea 100644 --- a/go.sum +++ b/go.sum @@ -800,6 +800,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go new file mode 100644 index 00000000..e2803125 --- /dev/null +++ b/pkg/crds/crds.go @@ -0,0 +1,160 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" + "github.com/rook/kubectl-rook-ceph/pkg/logging" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "strings" +) + +var cephResources = []string{ + "cephblockpoolradosnamespaces", + "cephblockpools", + "cephbucketnotifications", + "cephbuckettopics", + "cephclients", + "cephclusters", + "cephcosidrivers", + "cephfilesystemmirrors", + "cephfilesystems", + "cephfilesystemsubvolumegroup", + "cephnfses", + "cephobjectrealms", + "cephobjectstores", + "cephobjectstoreusers", + "cephobjectzonegroups", + "cephobjectzones", + "cephrbdmirrors", +} + +const ( + cephRookIoGroup = "ceph.rook.io" + cephRookResourcesVersion = "v1" +) + +const ( + CephResourceKindCephCluster = "CephCluster" +) + +var ( + errorResourceNotFound = errors.New("the server could not find the requested resource") + clusterResourcePatchFinalizer = map[string]interface{}{ + "spec": map[string]interface{}{ + "cleanupPolicy": map[string]string{ + "confirmation": "yes-really-destroy-data", + }, + }, + } +) + +func DeleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, clusterNamespace string) { + err := deleteCustomResources(ctx, clientsets, clusterNamespace) + if err != nil { + logging.Fatal(err) + } + logging.Info("done") +} + +func deleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, clusterNamespace string) error { + for _, resource := range cephResources { + logging.Info(fmt.Sprintf("Getting resources kind %s", resource)) + items, err := clientsets.ListResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace) + if err != nil { + if strings.Contains(err.Error(), errorResourceNotFound.Error()) { + logging.Warning("the server could not find the requested resource: %s", resource) + continue + } + // log instead of return + return err + } + + if len(items) == 0 { + logging.Warning("resource %s was not found on the cluster", resource) + continue + } + + for _, item := range items { + logging.Info(fmt.Sprintf("removing resource %s: %s", resource, item.GetName())) + err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName()) + if err != nil { + if err.Error() == "the server could not find the requested resource" { + logging.Info(err.Error()) + continue + } + // log instead of return + return err + } + + itemResource, err := clientsets.GetResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, item.GetName(), clusterNamespace) + if err != nil { + if err.Error() == "the server could not find the requested resource" { + logging.Info(err.Error()) + continue + } + // log instead of return + return err + } + + if itemResource != nil { + logging.Info("resource still alive, applying patch...") + err = updatingFinalizers(ctx, clientsets, itemResource, resource, clusterNamespace) + if err != nil { + if err.Error() == "the server could not find the requested resource" { + logging.Info(err.Error()) + continue + } + // log instead of return + return err + } + + err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName()) + if err != nil { + if err.Error() == "the server could not find the requested resource" { + logging.Info(err.Error()) + continue + } + // log instead of return + return err + } + } + logging.Info(fmt.Sprintf("resource %s was removed", item.GetName())) + } + } + return nil +} + +func updatingFinalizers(ctx context.Context, clientsets k8sutil.ClientsetsInterface, itemResource *unstructured.Unstructured, resource, clusterNamespace string) error { + patch := map[string]interface{}{} + + if itemResource.GetKind() == CephResourceKindCephCluster { + patch = clusterResourcePatchFinalizer + } + jsonPatchData, _ := json.Marshal(patch) + + err := clientsets.PatchResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, itemResource.GetName(), types.MergePatchType, jsonPatchData) + if err != nil { + return err + } + return nil +} diff --git a/pkg/crds/crds_test.go b/pkg/crds/crds_test.go new file mode 100644 index 00000000..4c0ea381 --- /dev/null +++ b/pkg/crds/crds_test.go @@ -0,0 +1,243 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "context" + "fmt" + "github.com/golang/mock/gomock" + "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "testing" +) + +func NewUnstructuredData(version, kind, name string) *unstructured.Unstructured { + obj := unstructured.Unstructured{} + obj.SetAPIVersion(version) + obj.SetKind(kind) + obj.SetName(name) + return &obj +} + +func TestDeleteCustomResources(t *testing.T) { + + type MockListResourcesDynamically struct { + items []unstructured.Unstructured + err error + } + + type MockGetResourceDynamically struct { + itemResource *unstructured.Unstructured + err error + } + + type given struct { + MockListResourcesDynamically MockListResourcesDynamically + MockGetResourceDynamically MockGetResourceDynamically + DeleteResourcesDynamicallyErr error + PatchResourcesDynamicallyErr error + } + + type expected struct { + err error + } + + var cases = []struct { + name string + given given + expected expected + }{ + { + name: "Should return error if was not able to run ListResourcesDynamically successfully", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + err: fmt.Errorf("error from ListResourcesDynamically"), + }, + }, + expected: expected{ + err: fmt.Errorf("error from ListResourcesDynamically"), + }, + }, + { + name: "Should return no error if the error from ListResourcesDynamically was resource not found", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + err: errorResourceNotFound, + }, + }, + }, + { + name: "Should return no error if no any error was thrown and the items from the list of resource is empty", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{}, + }, + }, + { + name: "Should return error if an error was throw by DeleteResourcesDynamically", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + DeleteResourcesDynamicallyErr: errorResourceNotFound, + }, + }, + { + name: "Should return no error if the error from GetResourcesDynamically was the server could not find the requested resource", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + err: fmt.Errorf("the server could not find the requested resource"), + }, + }, + }, + { + name: "Should return error if was unable to patch the resource kind cephblockpools", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"), + }, + expected: expected{ + err: fmt.Errorf("unable to patch the resource"), + }, + }, + { + name: "Should return error if was unable to patch the resource kind cephclusters with cleanupPolicy patch", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"), + }, + expected: expected{ + err: fmt.Errorf("unable to patch the resource"), + }, + }, + { + name: "Should return no error if was unable to patch the resource due the server could not find the requested resource", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + PatchResourcesDynamicallyErr: fmt.Errorf("the server could not find the requested resource"), + }, + }, + { + name: "Should return no error if DeleteResourcesDynamically returns the server could not find the requested resource ", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + DeleteResourcesDynamicallyErr: errorResourceNotFound, + }, + }, + { + name: "Should return no error if the resource was deleted successfully", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + clientSets := k8sutil.NewMockClientsetsInterface(ctrl) + clientSets. + EXPECT(). + ListResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.MockListResourcesDynamically.items, tc.given.MockListResourcesDynamically.err). + AnyTimes() + + clientSets. + EXPECT(). + GetResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.MockGetResourceDynamically.itemResource, tc.given.MockGetResourceDynamically.err). + AnyTimes() + + counter := 0 + + clientSets. + EXPECT(). + DeleteResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Do(func(arg1, arg2, arg3, arg4, arg5, arg6 interface{}) error { + if counter%2 == 0 { + return nil + } + return tc.given.DeleteResourcesDynamicallyErr + }). + Return(tc.given.DeleteResourcesDynamicallyErr). + AnyTimes() + + clientSets. + EXPECT(). + PatchResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.PatchResourcesDynamicallyErr). + AnyTimes() + + clusterNamespace := "rook-ceph" + err := DeleteCustomResources(context.Background(), clientSets, clusterNamespace) + if tc.expected.err != nil { + assert.Error(t, err) + assert.Equal(t, tc.expected.err.Error(), err.Error()) + return + } + + assert.NoError(t, err) + }) + } +} diff --git a/pkg/k8sutil/context.go b/pkg/k8sutil/context.go index 6878f97f..9a8b287f 100644 --- a/pkg/k8sutil/context.go +++ b/pkg/k8sutil/context.go @@ -17,6 +17,7 @@ limitations under the License. package k8sutil import ( + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -32,4 +33,7 @@ type Clientsets struct { // Rook is a typed connection to the rook API Rook rookclient.Interface + + // Dynamic is used for manage dynamic resources + Dynamic dynamic.Interface } diff --git a/pkg/k8sutil/dynamic.go b/pkg/k8sutil/dynamic.go new file mode 100644 index 00000000..3e4e09ce --- /dev/null +++ b/pkg/k8sutil/dynamic.go @@ -0,0 +1,121 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func (c *Clientsets) ListResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, +) ([]unstructured.Unstructured, error) { + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + list, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + List(ctx, metav1.ListOptions{}) + + if err != nil { + return nil, err + } + + return list.Items, nil +} + +func (c *Clientsets) DeleteResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, + resourceName string, +) error { + + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Delete(ctx, resourceName, metav1.DeleteOptions{}) + + if err != nil { + return err + } + return nil +} + +func (c *Clientsets) PatchResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, + resourceName string, + pt types.PatchType, + data []byte, +) error { + + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + _, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Patch(ctx, resourceName, pt, data, metav1.PatchOptions{}) + + if err != nil { + return err + } + return nil +} + +func (c *Clientsets) GetResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + name string, + namespace string, +) (*unstructured.Unstructured, error) { + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + item, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Get(ctx, name, metav1.GetOptions{}) + + if err != nil { + return nil, err + } + + return item, nil +} diff --git a/pkg/k8sutil/interface.go b/pkg/k8sutil/interface.go new file mode 100644 index 00000000..47596f19 --- /dev/null +++ b/pkg/k8sutil/interface.go @@ -0,0 +1,31 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "context" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +//go:generate mockgen -package=k8sutil --build_flags=--mod=mod -destination=mocks.go github.com/rook/kubectl-rook-ceph/pkg/k8sutil ClientsetsInterface +type ClientsetsInterface interface { + ListResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string) ([]unstructured.Unstructured, error) + GetResourcesDynamically(ctx context.Context, group string, version string, resource string, name string, namespace string) (*unstructured.Unstructured, error) + DeleteResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string) error + PatchResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string, pt types.PatchType, data []byte) error +} diff --git a/pkg/k8sutil/mocks.go b/pkg/k8sutil/mocks.go new file mode 100644 index 00000000..b6b55964 --- /dev/null +++ b/pkg/k8sutil/mocks.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/rook/kubectl-rook-ceph/pkg/k8sutil (interfaces: ClientsetsInterface) + +// Package k8sutil is a generated GoMock package. +package k8sutil + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "k8s.io/apimachinery/pkg/types" +) + +// MockClientsetsInterface is a mock of ClientsetsInterface interface. +type MockClientsetsInterface struct { + ctrl *gomock.Controller + recorder *MockClientsetsInterfaceMockRecorder +} + +// MockClientsetsInterfaceMockRecorder is the mock recorder for MockClientsetsInterface. +type MockClientsetsInterfaceMockRecorder struct { + mock *MockClientsetsInterface +} + +// NewMockClientsetsInterface creates a new mock instance. +func NewMockClientsetsInterface(ctrl *gomock.Controller) *MockClientsetsInterface { + mock := &MockClientsetsInterface{ctrl: ctrl} + mock.recorder = &MockClientsetsInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClientsetsInterface) EXPECT() *MockClientsetsInterfaceMockRecorder { + return m.recorder +} + +// DeleteResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) DeleteResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteResourcesDynamically indicates an expected call of DeleteResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) DeleteResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).DeleteResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// GetResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) GetResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string) (*unstructured.Unstructured, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(*unstructured.Unstructured) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResourcesDynamically indicates an expected call of GetResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) GetResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).GetResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// ListResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) ListResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4 string) ([]unstructured.Unstructured, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListResourcesDynamically", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].([]unstructured.Unstructured) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListResourcesDynamically indicates an expected call of ListResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) ListResourcesDynamically(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).ListResourcesDynamically), arg0, arg1, arg2, arg3, arg4) +} + +// PatchResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) PatchResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string, arg6 types.PatchType, arg7 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PatchResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret0, _ := ret[0].(error) + return ret0 +} + +// PatchResourcesDynamically indicates an expected call of PatchResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) PatchResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).PatchResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) +}