diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index 44eda192..3b69a9de 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -141,6 +141,20 @@ jobs: kubectl rook-ceph -n rook-ceph restore-deleted cephcluster my-cluster tests/github-action-helper.sh wait_for_crd_to_be_ready_default + - name: Show Cluster State + run: | + set -ex + kubectl -n rook-ceph get all + + - name: Destroy Cluster (removing CRs) + env: + ROOK_PLUGIN_SKIP_PROMPTS: true + run: | + set -ex + kubectl rook-ceph destroy-cluster + sleep 1 + kubectl get deployments -n rook-ceph --no-headers| wc -l | (read n && [ $n -le 1 ] || { echo "the crs could not be deleted"; exit 1;}) + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs @@ -283,6 +297,20 @@ jobs: kubectl rook-ceph --operator-namespace test-operator -n test-cluster restore-deleted cephcluster my-cluster tests/github-action-helper.sh wait_for_crd_to_be_ready_custom + - name: Show Cluster State + run: | + set -ex + kubectl -n test-cluster get all + + - name: Destroy Cluster (removing CRs) + env: + ROOK_PLUGIN_SKIP_PROMPTS: true + run: | + set -ex + kubectl rook-ceph --operator-namespace test-operator -n test-cluster destroy-cluster + sleep 1 + kubectl get deployments -n rook-ceph --no-headers| wc -l | (read n && [ $n -le 1 ] || { echo "the crs could not be deleted"; exit 1;}) + - name: collect common logs if: always() uses: ./.github/workflows/collect-logs diff --git a/Makefile b/Makefile index 8995311b..e61b4202 100644 --- a/Makefile +++ b/Makefile @@ -29,3 +29,8 @@ help : @echo "build : Create go binary." @echo "test : Runs unit tests" @echo "clean : Remove go binary file." + +codegen: + @echo "generating mocks..." + @go generate ./... + @echo completed \ No newline at end of file diff --git a/README.md b/README.md index c90ebded..824d0116 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Visit docs below for complete details about each command and their flags uses. 1. [Restore mon quorum](docs/mons.md#restore-quorum) 1. [Disaster Recovery](docs/dr-health.md) 1. [Restore deleted CRs](docs/crd.md) +1. [Destroy cluster](docs/destroy-cluster.md) ## Examples diff --git a/cmd/commands/destroy_cluster.go b/cmd/commands/destroy_cluster.go new file mode 100644 index 00000000..5718b88a --- /dev/null +++ b/cmd/commands/destroy_cluster.go @@ -0,0 +1,51 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + "github.com/rook/kubectl-rook-ceph/pkg/crds" + "github.com/rook/kubectl-rook-ceph/pkg/logging" + "github.com/rook/kubectl-rook-ceph/pkg/mons" + "github.com/spf13/cobra" +) + +const ( + destroyClusterQuestion = "Are you sure you want to destroy the cluster in namespace %q? If absolutely certain, enter: " + destroyClusterAnswer + destroyClusterAnswer = "yes-really-destroy-cluster" +) + +// DestroyClusterCmd represents the command for destroy cluster +var DestroyClusterCmd = &cobra.Command{ + Use: "destroy-cluster", + Short: "delete ALL data in the Rook cluster and all Rook CRs", + + Run: func(cmd *cobra.Command, args []string) { + ctx := cmd.Context() + var answer string + logging.Warning(destroyClusterQuestion, CephClusterNamespace) + fmt.Scanf("%s", &answer) + err := mons.PromptToContinueOrCancel("", destroyClusterAnswer, answer) + if err != nil { + logging.Fatal(fmt.Errorf("the response %q to confirm the cluster deletion", destroyClusterAnswer)) + } + + logging.Info("proceeding") + clientsets := GetClientsets(ctx) + crds.DeleteCustomResources(ctx, clientsets, clientsets.Kube, CephClusterNamespace) + }, +} diff --git a/cmd/commands/root.go b/cmd/commands/root.go index ede154d1..376353da 100644 --- a/cmd/commands/root.go +++ b/cmd/commands/root.go @@ -18,6 +18,7 @@ package command import ( "context" "fmt" + "k8s.io/client-go/dynamic" "regexp" "strings" @@ -102,6 +103,11 @@ func GetClientsets(ctx context.Context) *k8sutil.Clientsets { logging.Fatal(err) } + clientsets.Dynamic, err = dynamic.NewForConfig(clientsets.KubeConfig) + if err != nil { + logging.Fatal(err) + } + PreValidationCheck(ctx, clientsets, OperatorNamespace, CephClusterNamespace) return clientsets diff --git a/cmd/main.go b/cmd/main.go index df2313bc..a6a24906 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,6 +39,7 @@ func addcommands() { command.Health, command.DrCmd, command.RestoreCmd, + command.DestroyClusterCmd, command.SubvolumeCmd, ) } diff --git a/docs/destroy-cluster.md b/docs/destroy-cluster.md new file mode 100644 index 00000000..05ac0ec1 --- /dev/null +++ b/docs/destroy-cluster.md @@ -0,0 +1,69 @@ +# Destroying a Cluster + +When a cluster is no longer needed and needs to be torn down, Rook has a [Cleanup guide](https://rook.io/docs/rook/latest/Getting-Started/ceph-teardown/) with instructions to tear it down cleanly. While following that document is highly preferred, there is a command that will automate the cleanup of Rook after applications are removed. +The `destroy-cluster` command destroys a Rook cluster and deletes CRs (custom resources) by force removing any finalizers that may be preventing the cleanup. + +## !!! Warning !!! +**This command is not reversible**, and it will destroy your Rook cluster completely with guaranteed data loss. +Only use this command if you are 100% sure that the cluster and its data should be destroyed. + +## `destroy-cluster` command + +To destroy a cluster, run the command: +```bash +$ kubectl rook-ceph destroy-cluster +Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"? If absolutely certain, enter: yes-really-destroy-cluster +``` + +Any other response will cause the command to abort. + +## Example of destroying a cluster +```bash +$ kubectl rook-ceph -n rook-ceph destroy-cluster +Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"? If absolutely certain, enter: yes-really-destroy-cluster +yes-really-destroy-cluster +Info: proceeding +Info: Getting resource kind cephclusters +Info: removing resource cephclusters: my-cluster +Info: resource "my-cluster" is not yet deleted, applying patch to remove finalizer... +Info: resource my-cluster was deleted +Info: Getting resource kind cephblockpoolradosnamespaces +Info: resource cephblockpoolradosnamespaces was not found on the cluster +Info: Getting resource kind cephblockpools +Info: removing resource cephblockpools: builtin-mgr +Info: resource "builtin-mgr" is not yet deleted, applying patch to remove finalizer... +Info: resource builtin-mgr was deleted +Info: Getting resource kind cephbucketnotifications +Info: resource cephbucketnotifications was not found on the cluster +Info: Getting resource kind cephbuckettopics +Info: resource cephbuckettopics was not found on the cluster +Info: Getting resource kind cephclients +Info: resource cephclients was not found on the cluster +Info: Getting resource kind cephcosidrivers +Info: resource cephcosidrivers was not found on the cluster +Info: Getting resource kind cephfilesystemmirrors +Info: resource cephfilesystemmirrors was not found on the cluster +Info: Getting resource kind cephfilesystems +Info: removing resource cephfilesystems: myfs +Info: resource myfs was deleted +Info: Getting resource kind cephfilesystemsubvolumegroup +Info: the server could not find the requested resource: cephfilesystemsubvolumegroup +Info: Getting resource kind cephnfses +Info: resource cephnfses was not found on the cluster +Info: Getting resource kind cephobjectrealms +Info: resource cephobjectrealms was not found on the cluster +Info: Getting resource kind cephobjectstores +Info: removing resource cephobjectstores: my-store +Info: resource my-store was deleted +Info: Getting resource kind cephobjectstoreusers +Info: resource cephobjectstoreusers was not found on the cluster +Info: Getting resource kind cephobjectzonegroups +Info: resource cephobjectzonegroups was not found on the cluster +Info: Getting resource kind cephobjectzones +Info: resource cephobjectzones was not found on the cluster +Info: Getting resource kind cephrbdmirrors +Info: resource cephrbdmirrors was not found on the cluster +Info: removing deployment rook-ceph-tools +Info: waiting to clean up resources +Info: done +``` \ No newline at end of file diff --git a/docs/health.md b/docs/health.md index 3219c4a8..58c884ba 100644 --- a/docs/health.md +++ b/docs/health.md @@ -13,7 +13,7 @@ Health commands logs have three ways of logging: 1. `Info`: This is just a logging information for the users. 2. `Warning`: which mean there is some improvement required in the cluster. -3. `Error`: This requires immediate user attentions to get the cluster in healthy state. +3. `Error`: This requires immediate user attention to get the cluster in healthy state. ## Output diff --git a/go.mod b/go.mod index 45c859ce..58dafaef 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/fatih/color v1.16.0 + github.com/golang/mock v1.6.0 github.com/pkg/errors v0.9.1 github.com/rook/rook v1.12.9 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index ee647cd6..b959b8f7 100644 --- a/go.sum +++ b/go.sum @@ -319,6 +319,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go new file mode 100644 index 00000000..d8f45993 --- /dev/null +++ b/pkg/crds/crds.go @@ -0,0 +1,224 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "context" + "encoding/json" + "fmt" + "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" + "github.com/rook/kubectl-rook-ceph/pkg/logging" + corev1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +var cephResources = []string{ + "cephclusters", + "cephblockpoolradosnamespaces", + "cephblockpools", + "cephbucketnotifications", + "cephbuckettopics", + "cephclients", + "cephcosidrivers", + "cephfilesystemmirrors", + "cephfilesystems", + "cephfilesystemsubvolumegroups", + "cephnfses", + "cephobjectrealms", + "cephobjectstores", + "cephobjectstoreusers", + "cephobjectzonegroups", + "cephobjectzones", + "cephrbdmirrors", +} + +const ( + cephRookIoGroup = "ceph.rook.io" + cephRookResourcesVersion = "v1" +) + +const ( + CephResourceCephClusters = "cephclusters" + toolBoxDeployment = "rook-ceph-tools" +) + +var ( + clusterResourcePatchFinalizer = map[string]interface{}{ + "spec": map[string]interface{}{ + "cleanupPolicy": map[string]string{ + "confirmation": "yes-really-destroy-data", + }, + }, + } + + defaultResourceRemoveFinalizers = map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": nil, + }, + } +) + +func DeleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, k8sClientSet kubernetes.Interface, clusterNamespace string) { + err := deleteCustomResources(ctx, clientsets, clusterNamespace) + if err != nil { + logging.Fatal(err) + } + k8sutil.DeleteDeployment(ctx, k8sClientSet, clusterNamespace, toolBoxDeployment) + ensureClusterIsEmpty(ctx, k8sClientSet, clusterNamespace) + logging.Info("done") +} + +func deleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, clusterNamespace string) error { + for _, resource := range cephResources { + logging.Info("getting resource kind %s", resource) + items, err := clientsets.ListResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace) + if err != nil { + if k8sErrors.IsNotFound(err) { + logging.Info("the server could not find the requested resource: %s", resource) + continue + } + return err + } + + if len(items) == 0 { + logging.Info("resource %s was not found on the cluster", resource) + continue + } + + for _, item := range items { + logging.Info(fmt.Sprintf("removing resource %s: %s", resource, item.GetName())) + err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName()) + if err != nil { + if k8sErrors.IsNotFound(err) { + logging.Info(err.Error()) + continue + } + return err + } + + itemResource, err := clientsets.GetResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, item.GetName(), clusterNamespace) + if err != nil { + if !k8sErrors.IsNotFound(err) { + return err + } + } + + if itemResource != nil { + logging.Info(fmt.Sprintf("resource %q is not yet deleted, applying patch to remove finalizer...", itemResource.GetName())) + err = updatingFinalizers(ctx, clientsets, itemResource, resource, clusterNamespace) + if err != nil { + if k8sErrors.IsNotFound(err) { + logging.Info(err.Error()) + continue + } + return err + } + + err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName()) + if err != nil { + if !k8sErrors.IsNotFound(err) { + return err + } + } + } + + itemResource, err = clientsets.GetResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, item.GetName(), clusterNamespace) + if err != nil { + if !k8sErrors.IsNotFound(err) { + return err + } + } + + logging.Info("resource %s was deleted", item.GetName()) + } + } + return nil +} + +func updatingFinalizers(ctx context.Context, clientsets k8sutil.ClientsetsInterface, itemResource *unstructured.Unstructured, resource, clusterNamespace string) error { + if resource == CephResourceCephClusters { + jsonPatchData, _ := json.Marshal(clusterResourcePatchFinalizer) + err := clientsets.PatchResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, itemResource.GetName(), types.MergePatchType, jsonPatchData) + if err != nil { + return err + } + } + + jsonPatchData, _ := json.Marshal(defaultResourceRemoveFinalizers) + err := clientsets.PatchResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, itemResource.GetName(), types.MergePatchType, jsonPatchData) + if err != nil { + return err + } + + return nil +} + +func ensureClusterIsEmpty(ctx context.Context, k8sClientSet kubernetes.Interface, clusterNamespace string) { + logging.Info("waiting to clean up resources") + for { + pods, err := k8sClientSet.CoreV1().Pods(clusterNamespace).List(ctx, v1.ListOptions{LabelSelector: "rook_cluster=" + clusterNamespace}) + if err != nil { + logging.Fatal(err) + } + + if len(pods.Items) == 0 { + break + } + + logging.Info("%d pods still alive", len(pods.Items)) + for _, pod := range pods.Items { + appLabel := getPodLabel(ctx, pod, "app") + err := pruneDeployments(ctx, k8sClientSet, clusterNamespace, appLabel) + if err != nil { + logging.Fatal(err) + } + } + } +} + +func getPodLabel(_ context.Context, pod corev1.Pod, label string) string { + for podLabelName, podLabelValue := range pod.Labels { + if podLabelName == label { + return podLabelValue + } + } + return "" +} + +func pruneDeployments(ctx context.Context, k8sClientSet kubernetes.Interface, clusterNamespace, labelApp string) error { + deployments, err := k8sClientSet. + AppsV1(). + Deployments(clusterNamespace).List(ctx, v1.ListOptions{ + LabelSelector: "rook_cluster=" + clusterNamespace + ",app=" + labelApp, + }) + if err != nil { + if !k8sErrors.IsNotFound(err) { + return err + } + return nil + } + + for _, deployment := range deployments.Items { + logging.Info("deployment %s exists removing....\n", deployment.Name) + k8sutil.DeleteDeployment(ctx, k8sClientSet, clusterNamespace, deployment.Name) + } + return nil +} diff --git a/pkg/crds/crds_test.go b/pkg/crds/crds_test.go new file mode 100644 index 00000000..911bb310 --- /dev/null +++ b/pkg/crds/crds_test.go @@ -0,0 +1,245 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "context" + "fmt" + "github.com/golang/mock/gomock" + "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "testing" +) + +func NewUnstructuredData(version, kind, name string) *unstructured.Unstructured { + obj := unstructured.Unstructured{} + obj.SetAPIVersion(version) + obj.SetKind(kind) + obj.SetName(name) + return &obj +} + +func TestDeleteCustomResources(t *testing.T) { + + type MockListResourcesDynamically struct { + items []unstructured.Unstructured + err error + } + + type MockGetResourceDynamically struct { + itemResource *unstructured.Unstructured + err error + } + + type given struct { + MockListResourcesDynamically MockListResourcesDynamically + MockGetResourceDynamically MockGetResourceDynamically + DeleteResourcesDynamicallyErr error + PatchResourcesDynamicallyErr error + } + + type expected struct { + err error + } + + var cases = []struct { + name string + given given + expected expected + }{ + { + name: "Should return error if was not able to run ListResourcesDynamically successfully", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + err: fmt.Errorf("error from ListResourcesDynamically"), + }, + }, + expected: expected{ + err: fmt.Errorf("error from ListResourcesDynamically"), + }, + }, + { + name: "Should return no error if the error from ListResourcesDynamically was resource not found", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + err: errors.NewNotFound(schema.GroupResource{}, "cephrbdmirrors"), + }, + }, + }, + { + name: "Should return no error if no any error was thrown and the items from the list of resource is empty", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{}, + }, + }, + { + name: "Should return error if an error was throw by DeleteResourcesDynamically", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + DeleteResourcesDynamicallyErr: errors.NewNotFound(schema.GroupResource{}, "cephblockpools"), + }, + }, + { + name: "Should return no error if the error from GetResourcesDynamically was the server could not find the requested resource", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + err: errors.NewNotFound(schema.GroupResource{}, "cephblockpools"), + }, + }, + }, + { + name: "Should return error if was unable to patch the resource kind cephblockpools", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "cephblockpools", "cephblockpools"), + }, + PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"), + }, + expected: expected{ + err: fmt.Errorf("unable to patch the resource"), + }, + }, + { + name: "Should return error if was unable to patch the resource kind cephclusters with cleanupPolicy patch", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"), + }, + expected: expected{ + err: fmt.Errorf("unable to patch the resource"), + }, + }, + { + name: "Should return no error if was unable to patch the resource due the server could not find the requested resource", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + PatchResourcesDynamicallyErr: errors.NewNotFound(schema.GroupResource{}, "cephclusters"), + }, + }, + { + name: "Should return no error if DeleteResourcesDynamically returns the server could not find the requested resource ", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + DeleteResourcesDynamicallyErr: errors.NewNotFound(schema.GroupResource{}, "cephclusters"), + }, + }, + { + name: "Should return no error if the resource was deleted successfully", + given: given{ + MockListResourcesDynamically: MockListResourcesDynamically{ + items: []unstructured.Unstructured{ + *NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + MockGetResourceDynamically: MockGetResourceDynamically{ + itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"), + }, + }, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + clientSets := k8sutil.NewMockClientsetsInterface(ctrl) + clientSets. + EXPECT(). + ListResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.MockListResourcesDynamically.items, tc.given.MockListResourcesDynamically.err). + AnyTimes() + + clientSets. + EXPECT(). + GetResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.MockGetResourceDynamically.itemResource, tc.given.MockGetResourceDynamically.err). + AnyTimes() + + counter := 0 + + clientSets. + EXPECT(). + DeleteResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Do(func(arg1, arg2, arg3, arg4, arg5, arg6 interface{}) error { + if counter%2 == 1 { + return nil + } + return tc.given.DeleteResourcesDynamicallyErr + }). + Return(tc.given.DeleteResourcesDynamicallyErr). + AnyTimes() + + clientSets. + EXPECT(). + PatchResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(tc.given.PatchResourcesDynamicallyErr). + AnyTimes() + + clusterNamespace := "rook-ceph" + err := deleteCustomResources(context.Background(), clientSets, clusterNamespace) + if tc.expected.err != nil { + assert.Error(t, err) + assert.Equal(t, tc.expected.err.Error(), err.Error()) + return + } + + assert.NoError(t, err) + }) + } +} diff --git a/pkg/k8sutil/context.go b/pkg/k8sutil/context.go index 6878f97f..9a8b287f 100644 --- a/pkg/k8sutil/context.go +++ b/pkg/k8sutil/context.go @@ -17,6 +17,7 @@ limitations under the License. package k8sutil import ( + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -32,4 +33,7 @@ type Clientsets struct { // Rook is a typed connection to the rook API Rook rookclient.Interface + + // Dynamic is used for manage dynamic resources + Dynamic dynamic.Interface } diff --git a/pkg/k8sutil/dynamic.go b/pkg/k8sutil/dynamic.go new file mode 100644 index 00000000..3e4e09ce --- /dev/null +++ b/pkg/k8sutil/dynamic.go @@ -0,0 +1,121 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func (c *Clientsets) ListResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, +) ([]unstructured.Unstructured, error) { + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + list, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + List(ctx, metav1.ListOptions{}) + + if err != nil { + return nil, err + } + + return list.Items, nil +} + +func (c *Clientsets) DeleteResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, + resourceName string, +) error { + + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Delete(ctx, resourceName, metav1.DeleteOptions{}) + + if err != nil { + return err + } + return nil +} + +func (c *Clientsets) PatchResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + namespace string, + resourceName string, + pt types.PatchType, + data []byte, +) error { + + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + _, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Patch(ctx, resourceName, pt, data, metav1.PatchOptions{}) + + if err != nil { + return err + } + return nil +} + +func (c *Clientsets) GetResourcesDynamically( + ctx context.Context, + group string, + version string, + resource string, + name string, + namespace string, +) (*unstructured.Unstructured, error) { + resourceId := schema.GroupVersionResource{ + Group: group, + Version: version, + Resource: resource, + } + + item, err := c.Dynamic.Resource(resourceId).Namespace(namespace). + Get(ctx, name, metav1.GetOptions{}) + + if err != nil { + return nil, err + } + + return item, nil +} diff --git a/pkg/k8sutil/interface.go b/pkg/k8sutil/interface.go new file mode 100644 index 00000000..47596f19 --- /dev/null +++ b/pkg/k8sutil/interface.go @@ -0,0 +1,31 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8sutil + +import ( + "context" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +//go:generate mockgen -package=k8sutil --build_flags=--mod=mod -destination=mocks.go github.com/rook/kubectl-rook-ceph/pkg/k8sutil ClientsetsInterface +type ClientsetsInterface interface { + ListResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string) ([]unstructured.Unstructured, error) + GetResourcesDynamically(ctx context.Context, group string, version string, resource string, name string, namespace string) (*unstructured.Unstructured, error) + DeleteResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string) error + PatchResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string, pt types.PatchType, data []byte) error +} diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 9f469fd8..d8598422 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -19,6 +19,7 @@ package k8sutil import ( "context" "fmt" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" "time" "github.com/rook/kubectl-rook-ceph/pkg/logging" @@ -104,3 +105,16 @@ func GetDeployment(ctx context.Context, k8sclientset kubernetes.Interface, clust logging.Info("deployment %s exists\n", deploymentName) return deployment, nil } + +func DeleteDeployment(ctx context.Context, k8sclientset kubernetes.Interface, clusterNamespace string, deployment string) { + logging.Info("removing deployment %s", deployment) + + err := k8sclientset.AppsV1().Deployments(clusterNamespace).Delete(ctx, deployment, v1.DeleteOptions{}) + if err != nil { + if k8sErrors.IsNotFound(err) { + logging.Info("the server could not find the requested deployment: %s", deployment) + return + } + logging.Fatal(err) + } +} diff --git a/pkg/k8sutil/mocks.go b/pkg/k8sutil/mocks.go new file mode 100644 index 00000000..b6b55964 --- /dev/null +++ b/pkg/k8sutil/mocks.go @@ -0,0 +1,95 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/rook/kubectl-rook-ceph/pkg/k8sutil (interfaces: ClientsetsInterface) + +// Package k8sutil is a generated GoMock package. +package k8sutil + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + types "k8s.io/apimachinery/pkg/types" +) + +// MockClientsetsInterface is a mock of ClientsetsInterface interface. +type MockClientsetsInterface struct { + ctrl *gomock.Controller + recorder *MockClientsetsInterfaceMockRecorder +} + +// MockClientsetsInterfaceMockRecorder is the mock recorder for MockClientsetsInterface. +type MockClientsetsInterfaceMockRecorder struct { + mock *MockClientsetsInterface +} + +// NewMockClientsetsInterface creates a new mock instance. +func NewMockClientsetsInterface(ctrl *gomock.Controller) *MockClientsetsInterface { + mock := &MockClientsetsInterface{ctrl: ctrl} + mock.recorder = &MockClientsetsInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClientsetsInterface) EXPECT() *MockClientsetsInterfaceMockRecorder { + return m.recorder +} + +// DeleteResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) DeleteResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteResourcesDynamically indicates an expected call of DeleteResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) DeleteResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).DeleteResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// GetResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) GetResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string) (*unstructured.Unstructured, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(*unstructured.Unstructured) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResourcesDynamically indicates an expected call of GetResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) GetResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).GetResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// ListResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) ListResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4 string) ([]unstructured.Unstructured, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListResourcesDynamically", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].([]unstructured.Unstructured) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListResourcesDynamically indicates an expected call of ListResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) ListResourcesDynamically(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).ListResourcesDynamically), arg0, arg1, arg2, arg3, arg4) +} + +// PatchResourcesDynamically mocks base method. +func (m *MockClientsetsInterface) PatchResourcesDynamically(arg0 context.Context, arg1, arg2, arg3, arg4, arg5 string, arg6 types.PatchType, arg7 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PatchResourcesDynamically", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret0, _ := ret[0].(error) + return ret0 +} + +// PatchResourcesDynamically indicates an expected call of PatchResourcesDynamically. +func (mr *MockClientsetsInterfaceMockRecorder) PatchResourcesDynamically(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PatchResourcesDynamically", reflect.TypeOf((*MockClientsetsInterface)(nil).PatchResourcesDynamically), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) +}