Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
commands: add command to wiping cluster
Browse files Browse the repository at this point in the history
Added command destroy-cluster for remove all crds
Added documentation for destroy-cluster
Added unittest and integration tests

Signed-off-by: Javier <sjavierlopez@gmail.com>
Javlopez committed Oct 3, 2023
1 parent 7fb9e7d commit d76512c
Showing 16 changed files with 901 additions and 1 deletion.
17 changes: 17 additions & 0 deletions .github/workflows/go-test.yaml
Original file line number Diff line number Diff line change
@@ -225,6 +225,23 @@ jobs:
kubectl -n test-cluster scale deployment rook-ceph-osd-0 --replicas 0
kubectl rook-ceph --operator-namespace test-operator -n test-cluster rook purge-osd 0 --force
- name: List CRDS
run: |
set -ex
kubectl -n test-cluster get all
- name: Destroy Cluster (removing CRDS)
env:
ROOK_PLUGIN_SKIP_PROMPTS: true
run: |
set -ex
kubectl rook-ceph --operator-namespace test-operator -n test-cluster destroy-cluster
- name: Validate destroyed CRDS
run: |
set -ex
kubectl get deployments -n rook-ceph --no-headers| wc -l | (read n && [ $n -le 1 ] || { echo "the crds could not be deleted"; exit 1;})
- name: collect common logs
if: always()
uses: ./.github/workflows/collect-logs
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -29,3 +29,8 @@ help :
@echo "build : Create go binary."
@echo "test : Runs unit tests"
@echo "clean : Remove go binary file."

generate:
@echo "generating mocks..."
@go generate ./...
@echo completed
44 changes: 44 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -101,6 +101,7 @@ Visit docs below for complete details about each command and their flags uses.
1. [Debug OSDs and Mons](docs/debug.md)
1. [Restore mon quorum](docs/mons.md#restore-quorum)
1. [Disaster Recovery](docs/dr-health.md)
1. [Destroy Cluster](docs/destroy_cluster.md)

## Examples

@@ -178,6 +179,49 @@ kubectl rook-ceph ceph versions
}
}
```
### Destroy Cluster
```bash
$ kubectl rook-ceph -n rook-ceph destroy-cluster
Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"?
yes-really-destroy-cluster
Info: proceeding
Info: Getting resources kind cephblockpoolradosnamespaces
Warning: resource cephblockpoolradosnamespaces was not found on the cluster
Info: Getting resources kind cephblockpools
Warning: resource cephblockpools was not found on the cluster
Info: Getting resources kind cephbucketnotifications
Warning: resource cephbucketnotifications was not found on the cluster
Info: Getting resources kind cephbuckettopics
Warning: resource cephbuckettopics was not found on the cluster
Info: Getting resources kind cephclients
Warning: resource cephclients was not found on the cluster
Info: Getting resources kind cephclusters
Warning: resource cephclusters was not found on the cluster
Info: Getting resources kind cephcosidrivers
Warning: resource cephcosidrivers was not found on the cluster
Info: Getting resources kind cephfilesystemmirrors
Warning: resource cephfilesystemmirrors was not found on the cluster
Info: Getting resources kind cephfilesystems
Warning: resource cephfilesystems was not found on the cluster
Info: Getting resources kind cephfilesystemsubvolumegroup
Warning: the server could not find the requested resource: cephfilesystemsubvolumegroup
Info: Getting resources kind cephnfses
Warning: resource cephnfses was not found on the cluster
Info: Getting resources kind cephobjectrealms
Warning: resource cephobjectrealms was not found on the cluster
Info: Getting resources kind cephobjectstores
Warning: resource cephobjectstores was not found on the cluster
Info: Getting resources kind cephobjectstoreusers
Warning: resource cephobjectstoreusers was not found on the cluster
Info: Getting resources kind cephobjectzonegroups
Warning: resource cephobjectzonegroups was not found on the cluster
Info: Getting resources kind cephobjectzones
Warning: resource cephobjectzones was not found on the cluster
Info: Getting resources kind cephrbdmirrors
Warning: resource cephrbdmirrors was not found on the cluster
Info: done
$
```

## Contributing

63 changes: 63 additions & 0 deletions cmd/commands/destroy_cluster.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package command

import (
"fmt"
"github.com/rook/kubectl-rook-ceph/pkg/crds"
"github.com/rook/kubectl-rook-ceph/pkg/logging"
"github.com/spf13/cobra"
"os"
"strings"
)

const (
destroyClusterQuestion = "Are you sure you want to destroy the cluster in namespace %q? If absolutely certain, enter: " + destroyClusterAnswer
destroyClusterAnswer = "yes-really-destroy-cluster"
)

// DestroyClusterCmd represents the command for destroy cluster
var DestroyClusterCmd = &cobra.Command{
Use: "destroy-cluster",
Short: "delete all Rook CRDs",
Run: func(cmd *cobra.Command, args []string) {
question := fmt.Sprintf(destroyClusterQuestion, CephClusterNamespace)
confirmation, err := confirmPrompt(question, destroyClusterAnswer)
if err != nil {
logging.Fatal(err)
}

logging.Info(confirmation)
clientsets := GetClientsets(cmd.Context())
crds.DeleteCustomResources(cmd.Context(), clientsets, CephClusterNamespace)
},
}

func confirmPrompt(question, key string) (string, error) {
var answer string
if skip, ok := os.LookupEnv("ROOK_PLUGIN_SKIP_PROMPTS"); ok && skip == "true" {
return "skipped prompt since ROOK_PLUGIN_SKIP_PROMPTS=true", nil
}
logging.Warning(question)
fmt.Scan(&answer)

if strings.EqualFold(answer, key) {
return "proceeding", nil
}

return "", fmt.Errorf("the response %q to confirm the cluster deletion", destroyClusterAnswer)
}
6 changes: 6 additions & 0 deletions cmd/commands/root.go
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ package command
import (
"context"
"fmt"
"k8s.io/client-go/dynamic"
"regexp"
"strings"

@@ -102,6 +103,11 @@ func GetClientsets(ctx context.Context) *k8sutil.Clientsets {
logging.Fatal(err)
}

clientsets.Dynamic, err = dynamic.NewForConfig(clientsets.KubeConfig)
if err != nil {
logging.Fatal(err)
}

PreValidationCheck(ctx, clientsets, OperatorNamespace, CephClusterNamespace)

return clientsets
1 change: 1 addition & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
@@ -38,5 +38,6 @@ func addcommands() {
command.DebugCmd,
command.Health,
command.DrCmd,
command.DestroyClusterCmd,
)
}
108 changes: 108 additions & 0 deletions docs/destroy_cluster.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
# Destroy Cluster

This command is used for destroy CRDS (custom resource definitions) created by rook-ceph

## !!! Warning !!!
**This command is not reversible**, and it will destroy your rook-ceph cluster completely and your data,
please only use this command if your sure that your data must be destroyed.

## How to use:

Just you need to run the command, and you will be asked for the confirmation
```bash
$ kubectl rook-ceph -n rook-ceph destroy-cluster
Are you sure you want to destroy the cluster in namespace "rook-ceph"? |
```

You must write exactly this answer **yes-really-destroy-cluster**
to confirm that you really want to execute the command and destroy
your cluster, any other response will be rejected and
the process will be stopped even responding yes

```bash
$ kubectl rook-ceph -n rook-ceph destroy-cluster
Are you sure you want to destroy the cluster in namespace "rook-ceph"? yes
Error: you need to response "yes-really-destroy-cluster" to confirm the cluster deletion
exit status 1

```

Once you respond with the right answer the process will start, and it will be deleting the CRDS created by rook-ceph

```bash
$ kubectl rook-ceph -n rook-ceph destroy-cluster
Warning: Are you sure you want to destroy the cluster in namespace "rook-ceph"?
yes-really-destroy-cluster
Info: proceeding
Info: Getting resources kind cephblockpoolradosnamespaces
Warning: resource cephblockpoolradosnamespaces was not found on the cluster
Info: Getting resources kind cephblockpools
Warning: resource cephblockpools was not found on the cluster
Info: Getting resources kind cephbucketnotifications
Warning: resource cephbucketnotifications was not found on the cluster
Info: Getting resources kind cephbuckettopics
Warning: resource cephbuckettopics was not found on the cluster
Info: Getting resources kind cephclients
Warning: resource cephclients was not found on the cluster
Info: Getting resources kind cephclusters
Warning: resource cephclusters was not found on the cluster
Info: Getting resources kind cephcosidrivers
Warning: resource cephcosidrivers was not found on the cluster
Info: Getting resources kind cephfilesystemmirrors
Warning: resource cephfilesystemmirrors was not found on the cluster
Info: Getting resources kind cephfilesystems
Warning: resource cephfilesystems was not found on the cluster
Info: Getting resources kind cephfilesystemsubvolumegroup
Warning: the server could not find the requested resource: cephfilesystemsubvolumegroup
Info: Getting resources kind cephnfses
Warning: resource cephnfses was not found on the cluster
Info: Getting resources kind cephobjectrealms
Warning: resource cephobjectrealms was not found on the cluster
Info: Getting resources kind cephobjectstores
Warning: resource cephobjectstores was not found on the cluster
Info: Getting resources kind cephobjectstoreusers
Warning: resource cephobjectstoreusers was not found on the cluster
Info: Getting resources kind cephobjectzonegroups
Warning: resource cephobjectzonegroups was not found on the cluster
Info: Getting resources kind cephobjectzones
Warning: resource cephobjectzones was not found on the cluster
Info: Getting resources kind cephrbdmirrors
Warning: resource cephrbdmirrors was not found on the cluster
Info: done
$
```


# Developer changes
We are adding unittests in order to ensure that the destroy-cluster command is working as we expected,
so if you want to expand the functionality, and you need to change the `pgk/k8sutil/interface.go` remember to recreate the mocks by using `make generate`

```bash
$ make generate
generating mocks...
completed
$
```

# Run test

To run the tests you should to execute `make test`
```bash
$ make test
running unit tests
go test ./...
? github.com/rook/kubectl-rook-ceph/cmd [no test files]
ok github.com/rook/kubectl-rook-ceph/cmd/commands (cached)
ok github.com/rook/kubectl-rook-ceph/pkg/crds (cached)
? github.com/rook/kubectl-rook-ceph/pkg/debug [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/dr [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/exec [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/health [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/k8sutil [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/logging [no test files]
? github.com/rook/kubectl-rook-ceph/pkg/rook [no test files]
ok github.com/rook/kubectl-rook-ceph/pkg/mons (cached)
$
```


2 changes: 1 addition & 1 deletion docs/health.md
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ Health commands logs have three ways of logging:

1. `Info`: This is just a logging information for the users.
2. `Warning`: which mean there is some improvement required in the cluster.
3. `Error`: This reuires immediate user attentions to get the cluster in healthy state.
3. `Error`: This requires immediate user attentions to get the cluster in healthy state.

## Output

1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ go 1.20

require (
github.com/fatih/color v1.15.0
github.com/golang/mock v1.6.0
github.com/rook/rook v1.12.2
github.com/rook/rook/pkg/apis v0.0.0-20230725213142-5979b3816292
github.com/spf13/cobra v1.7.0
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -800,6 +800,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
160 changes: 160 additions & 0 deletions pkg/crds/crds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package crds

import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
"github.com/rook/kubectl-rook-ceph/pkg/logging"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"strings"
)

var cephResources = []string{
"cephblockpoolradosnamespaces",
"cephblockpools",
"cephbucketnotifications",
"cephbuckettopics",
"cephclients",
"cephclusters",
"cephcosidrivers",
"cephfilesystemmirrors",
"cephfilesystems",
"cephfilesystemsubvolumegroup",
"cephnfses",
"cephobjectrealms",
"cephobjectstores",
"cephobjectstoreusers",
"cephobjectzonegroups",
"cephobjectzones",
"cephrbdmirrors",
}

const (
cephRookIoGroup = "ceph.rook.io"
cephRookResourcesVersion = "v1"
)

const (
CephResourceKindCephCluster = "CephCluster"
)

var (
errorResourceNotFound = errors.New("the server could not find the requested resource")
clusterResourcePatchFinalizer = map[string]interface{}{
"spec": map[string]interface{}{
"cleanupPolicy": map[string]string{
"confirmation": "yes-really-destroy-data",
},
},
}
)

func DeleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, clusterNamespace string) {
err := deleteCustomResources(ctx, clientsets, clusterNamespace)
if err != nil {
logging.Fatal(err)
}
logging.Info("done")
}

func deleteCustomResources(ctx context.Context, clientsets k8sutil.ClientsetsInterface, clusterNamespace string) error {
for _, resource := range cephResources {
logging.Info(fmt.Sprintf("Getting resources kind %s", resource))
items, err := clientsets.ListResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace)
if err != nil {
if strings.Contains(err.Error(), errorResourceNotFound.Error()) {
logging.Warning("the server could not find the requested resource: %s", resource)
continue
}
// log instead of return
return err
}

if len(items) == 0 {
logging.Warning("resource %s was not found on the cluster", resource)
continue
}

for _, item := range items {
logging.Info(fmt.Sprintf("removing resource %s: %s", resource, item.GetName()))
err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName())
if err != nil {
if err.Error() == "the server could not find the requested resource" {
logging.Info(err.Error())
continue
}
// log instead of return
return err
}

itemResource, err := clientsets.GetResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, item.GetName(), clusterNamespace)
if err != nil {
if err.Error() == "the server could not find the requested resource" {
logging.Info(err.Error())
continue
}
// log instead of return
return err
}

if itemResource != nil {
logging.Info("resource still alive, applying patch...")
err = updatingFinalizers(ctx, clientsets, itemResource, resource, clusterNamespace)
if err != nil {
if err.Error() == "the server could not find the requested resource" {
logging.Info(err.Error())
continue
}
// log instead of return
return err
}

err = clientsets.DeleteResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, item.GetName())
if err != nil {
if err.Error() == "the server could not find the requested resource" {
logging.Info(err.Error())
continue
}
// log instead of return
return err
}
}
logging.Info(fmt.Sprintf("resource %s was removed", item.GetName()))
}
}
return nil
}

func updatingFinalizers(ctx context.Context, clientsets k8sutil.ClientsetsInterface, itemResource *unstructured.Unstructured, resource, clusterNamespace string) error {
patch := map[string]interface{}{}

if itemResource.GetKind() == CephResourceKindCephCluster {
patch = clusterResourcePatchFinalizer
}
jsonPatchData, _ := json.Marshal(patch)

err := clientsets.PatchResourcesDynamically(ctx, cephRookIoGroup, cephRookResourcesVersion, resource, clusterNamespace, itemResource.GetName(), types.MergePatchType, jsonPatchData)
if err != nil {
return err
}
return nil
}
243 changes: 243 additions & 0 deletions pkg/crds/crds_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package crds

import (
"context"
"fmt"
"github.com/golang/mock/gomock"
"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
"github.com/stretchr/testify/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"testing"
)

func NewUnstructuredData(version, kind, name string) *unstructured.Unstructured {
obj := unstructured.Unstructured{}
obj.SetAPIVersion(version)
obj.SetKind(kind)
obj.SetName(name)
return &obj
}

func TestDeleteCustomResources(t *testing.T) {

type MockListResourcesDynamically struct {
items []unstructured.Unstructured
err error
}

type MockGetResourceDynamically struct {
itemResource *unstructured.Unstructured
err error
}

type given struct {
MockListResourcesDynamically MockListResourcesDynamically
MockGetResourceDynamically MockGetResourceDynamically
DeleteResourcesDynamicallyErr error
PatchResourcesDynamicallyErr error
}

type expected struct {
err error
}

var cases = []struct {
name string
given given
expected expected
}{
{
name: "Should return error if was not able to run ListResourcesDynamically successfully",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
err: fmt.Errorf("error from ListResourcesDynamically"),
},
},
expected: expected{
err: fmt.Errorf("error from ListResourcesDynamically"),
},
},
{
name: "Should return no error if the error from ListResourcesDynamically was resource not found",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
err: errorResourceNotFound,
},
},
},
{
name: "Should return no error if no any error was thrown and the items from the list of resource is empty",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{},
},
},
{
name: "Should return error if an error was throw by DeleteResourcesDynamically",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "cephblockpools", "cephblockpools"),
},
},
DeleteResourcesDynamicallyErr: errorResourceNotFound,
},
},
{
name: "Should return no error if the error from GetResourcesDynamically was the server could not find the requested resource",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "cephblockpools", "cephblockpools"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
err: fmt.Errorf("the server could not find the requested resource"),
},
},
},
{
name: "Should return error if was unable to patch the resource kind cephblockpools",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "cephblockpools", "cephblockpools"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
itemResource: NewUnstructuredData("v1", "cephblockpools", "cephblockpools"),
},
PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"),
},
expected: expected{
err: fmt.Errorf("unable to patch the resource"),
},
},
{
name: "Should return error if was unable to patch the resource kind cephclusters with cleanupPolicy patch",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
PatchResourcesDynamicallyErr: fmt.Errorf("unable to patch the resource"),
},
expected: expected{
err: fmt.Errorf("unable to patch the resource"),
},
},
{
name: "Should return no error if was unable to patch the resource due the server could not find the requested resource",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
PatchResourcesDynamicallyErr: fmt.Errorf("the server could not find the requested resource"),
},
},
{
name: "Should return no error if DeleteResourcesDynamically returns the server could not find the requested resource ",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
DeleteResourcesDynamicallyErr: errorResourceNotFound,
},
},
{
name: "Should return no error if the resource was deleted successfully",
given: given{
MockListResourcesDynamically: MockListResourcesDynamically{
items: []unstructured.Unstructured{
*NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
},
MockGetResourceDynamically: MockGetResourceDynamically{
itemResource: NewUnstructuredData("v1", "CephCluster", "cephclusters"),
},
},
},
}

for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

ctrl := gomock.NewController(t)
defer ctrl.Finish()

clientSets := k8sutil.NewMockClientsetsInterface(ctrl)
clientSets.
EXPECT().
ListResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Return(tc.given.MockListResourcesDynamically.items, tc.given.MockListResourcesDynamically.err).
AnyTimes()

clientSets.
EXPECT().
GetResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Return(tc.given.MockGetResourceDynamically.itemResource, tc.given.MockGetResourceDynamically.err).
AnyTimes()

counter := 0

clientSets.
EXPECT().
DeleteResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Do(func(arg1, arg2, arg3, arg4, arg5, arg6 interface{}) error {
if counter%2 == 0 {
return nil
}
return tc.given.DeleteResourcesDynamicallyErr
}).
Return(tc.given.DeleteResourcesDynamicallyErr).
AnyTimes()

clientSets.
EXPECT().
PatchResourcesDynamically(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Return(tc.given.PatchResourcesDynamicallyErr).
AnyTimes()

clusterNamespace := "rook-ceph"
err := DeleteCustomResources(context.Background(), clientSets, clusterNamespace)
if tc.expected.err != nil {
assert.Error(t, err)
assert.Equal(t, tc.expected.err.Error(), err.Error())
return
}

assert.NoError(t, err)
})
}
}
4 changes: 4 additions & 0 deletions pkg/k8sutil/context.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ limitations under the License.
package k8sutil

import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

@@ -32,4 +33,7 @@ type Clientsets struct {

// Rook is a typed connection to the rook API
Rook rookclient.Interface

// Dynamic is used for manage dynamic resources
Dynamic dynamic.Interface
}
121 changes: 121 additions & 0 deletions pkg/k8sutil/dynamic.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sutil

import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
)

func (c *Clientsets) ListResourcesDynamically(
ctx context.Context,
group string,
version string,
resource string,
namespace string,
) ([]unstructured.Unstructured, error) {
resourceId := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}

list, err := c.Dynamic.Resource(resourceId).Namespace(namespace).
List(ctx, metav1.ListOptions{})

if err != nil {
return nil, err
}

return list.Items, nil
}

func (c *Clientsets) DeleteResourcesDynamically(
ctx context.Context,
group string,
version string,
resource string,
namespace string,
resourceName string,
) error {

resourceId := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}
err := c.Dynamic.Resource(resourceId).Namespace(namespace).
Delete(ctx, resourceName, metav1.DeleteOptions{})

if err != nil {
return err
}
return nil
}

func (c *Clientsets) PatchResourcesDynamically(
ctx context.Context,
group string,
version string,
resource string,
namespace string,
resourceName string,
pt types.PatchType,
data []byte,
) error {

resourceId := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}

_, err := c.Dynamic.Resource(resourceId).Namespace(namespace).
Patch(ctx, resourceName, pt, data, metav1.PatchOptions{})

if err != nil {
return err
}
return nil
}

func (c *Clientsets) GetResourcesDynamically(
ctx context.Context,
group string,
version string,
resource string,
name string,
namespace string,
) (*unstructured.Unstructured, error) {
resourceId := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}

item, err := c.Dynamic.Resource(resourceId).Namespace(namespace).
Get(ctx, name, metav1.GetOptions{})

if err != nil {
return nil, err
}

return item, nil
}
31 changes: 31 additions & 0 deletions pkg/k8sutil/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sutil

import (
"context"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
)

//go:generate mockgen -package=k8sutil --build_flags=--mod=mod -destination=mocks.go github.com/rook/kubectl-rook-ceph/pkg/k8sutil ClientsetsInterface
type ClientsetsInterface interface {
ListResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string) ([]unstructured.Unstructured, error)
GetResourcesDynamically(ctx context.Context, group string, version string, resource string, name string, namespace string) (*unstructured.Unstructured, error)
DeleteResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string) error
PatchResourcesDynamically(ctx context.Context, group string, version string, resource string, namespace string, resourceName string, pt types.PatchType, data []byte) error
}
95 changes: 95 additions & 0 deletions pkg/k8sutil/mocks.go

0 comments on commit d76512c

Please sign in to comment.