diff --git a/e2e/cephfs.go b/e2e/cephfs.go index f4366c1d2313..310a557fa950 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -215,6 +215,11 @@ var _ = Describe(cephfsType, func() { if err != nil { framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, cephFSDeploymentName, err) } + + err = createSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) AfterEach(func() { @@ -263,6 +268,11 @@ var _ = Describe(cephfsType, func() { } } } + + err = deleteSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) Context("Test CephFS CSI", func() { @@ -939,13 +949,15 @@ var _ = Describe(cephfsType, func() { } // re-define configmap with information of multiple clusters. + subvolgrp1 := "subvolgrp1" + subvolgrp2 := "subvolgrp2" clusterInfo := map[string]map[string]string{} clusterID1 := "clusterID-1" clusterID2 := "clusterID-2" clusterInfo[clusterID1] = map[string]string{} - clusterInfo[clusterID1]["subvolumeGroup"] = "subvolgrp1" + clusterInfo[clusterID1]["subvolumeGroup"] = subvolgrp1 clusterInfo[clusterID2] = map[string]string{} - clusterInfo[clusterID2]["subvolumeGroup"] = "subvolgrp2" + clusterInfo[clusterID2]["subvolumeGroup"] = subvolgrp2 err = createCustomConfigMap(f.ClientSet, cephFSDirPath, clusterInfo) if err != nil { @@ -966,11 +978,20 @@ var _ = Describe(cephfsType, func() { if err != nil { framework.Failf("failed to delete storageclass: %v", err) } + + err = createSubvolumegroup(f, fileSystemName, subvolgrp1) + if err != nil { + framework.Failf("%v", err) + } // verify subvolume group creation. - err = validateSubvolumegroup(f, "subvolgrp1") + err = validateSubvolumegroup(f, subvolgrp1) if err != nil { framework.Failf("failed to validate subvolume group: %v", err) } + err = deleteSubvolumegroup(f, fileSystemName, subvolgrp1) + if err != nil { + framework.Failf("%v", err) + } // create resources and verify subvolume group creation // for the second cluster. @@ -987,10 +1008,19 @@ var _ = Describe(cephfsType, func() { if err != nil { framework.Failf("failed to delete storageclass: %v", err) } - err = validateSubvolumegroup(f, "subvolgrp2") + + err = createSubvolumegroup(f, fileSystemName, subvolgrp2) + if err != nil { + framework.Failf("%v", err) + } + err = validateSubvolumegroup(f, subvolgrp2) if err != nil { framework.Failf("failed to validate subvolume group: %v", err) } + err = deleteSubvolumegroup(f, fileSystemName, subvolgrp2) + if err != nil { + framework.Failf("%v", err) + } err = deleteConfigMap(cephFSDirPath) if err != nil { framework.Failf("failed to delete configmap: %v", err) diff --git a/e2e/configmap.go b/e2e/configmap.go index 6eb3660653f6..df23da1f6d83 100644 --- a/e2e/configmap.go +++ b/e2e/configmap.go @@ -66,6 +66,11 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra }} if upgradeTesting { subvolumegroup = "csi" + + err = createSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } } conmap[0].CephFS.SubvolumeGroup = subvolumegroup data, err := json.Marshal(conmap) diff --git a/e2e/nfs.go b/e2e/nfs.go index 56f9cc7f434c..5114990495f5 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -276,6 +276,11 @@ var _ = Describe("nfs", func() { if err != nil { framework.Failf("failed to create node secret: %v", err) } + + err = createSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) AfterEach(func() { @@ -322,6 +327,11 @@ var _ = Describe("nfs", func() { } } } + + err = deleteSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) Context("Test NFS CSI", func() { diff --git a/e2e/utils.go b/e2e/utils.go index 67e413f5e384..5a1a56939bd3 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1853,3 +1853,31 @@ func checkExports(f *framework.Framework, clusterID, clientString string) bool { return true } + +// createSubvolumegroup creates a subvolumegroup. +func createSubvolumegroup(f *framework.Framework, fileSystemName, subvolumegroup string) error { + cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fileSystemName, subvolumegroup) + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return fmt.Errorf("failed to exec command in toolbox: %w", err) + } + if stdErr != "" { + return fmt.Errorf("failed to create subvolumegroup %s : %v", subvolumegroup, stdErr) + } + + return nil +} + +// deleteSubvolumegroup creates a subvolumegroup +func deleteSubvolumegroup(f *framework.Framework, fileSystemName, subvolumegroup string) error { + cmd := fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fileSystemName, subvolumegroup) + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return fmt.Errorf("failed to exec command in toolbox: %w", err) + } + if stdErr != "" { + return fmt.Errorf("failed to remove subvolumegroup %s : %v", subvolumegroup, stdErr) + } + + return nil +} diff --git a/internal/cephfs/core/metadata.go b/internal/cephfs/core/metadata.go index 1dfd1b5c4e8e..9e2b90d5f468 100644 --- a/internal/cephfs/core/metadata.go +++ b/internal/cephfs/core/metadata.go @@ -51,32 +51,6 @@ func (s *subVolumeClient) isUnsupportedSubVolMetadata(err error) bool { return true } -// isSubVolumeGroupCreated returns true if subvolume group is created. -func (s *subVolumeClient) isSubVolumeGroupCreated() bool { - newLocalClusterState(s.clusterID) - clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RLock() - defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RUnlock() - - if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil { - return false - } - - return clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup] -} - -// updateSubVolumeGroupCreated updates subvolume group created map. -// If the map is nil, it creates a new map and updates it. -func (s *subVolumeClient) updateSubVolumeGroupCreated(state bool) { - clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Lock() - defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Unlock() - - if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil { - clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated = make(map[string]bool) - } - - clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup] = state -} - // setMetadata sets custom metadata on the subvolume in a volume as a // key-value pair. func (s *subVolumeClient) setMetadata(key, value string) error { diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index 0754b8a23f38..a92025ec93d9 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -210,14 +210,6 @@ type localClusterState struct { resizeState operationState subVolMetadataState operationState subVolSnapshotMetadataState operationState - // A cluster can have multiple filesystem for that we need to have a map of - // subvolumegroups to check filesystem is created nor not. - // set true once a subvolumegroup is created - // for corresponding filesystem in a cluster. - subVolumeGroupsCreated map[string]bool - // subVolumeGroupsRWMutex is used to protect subVolumeGroupsCreated map - // against concurrent writes while allowing multiple readers. - subVolumeGroupsRWMutex sync.RWMutex } func newLocalClusterState(clusterID string) { @@ -241,24 +233,6 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { return err } - // create subvolumegroup if not already created for the cluster. - if !s.isSubVolumeGroupCreated() { - opts := fsAdmin.SubVolumeGroupOptions{} - err = ca.CreateSubVolumeGroup(s.FsName, s.SubvolumeGroup, &opts) - if err != nil { - log.ErrorLog( - ctx, - "failed to create subvolume group %s, for the vol %s: %s", - s.SubvolumeGroup, - s.VolID, - err) - - return err - } - log.DebugLog(ctx, "cephfs: created subvolume group %s", s.SubvolumeGroup) - s.updateSubVolumeGroupCreated(true) - } - opts := fsAdmin.SubVolumeOptions{ Size: fsAdmin.ByteCount(s.Size), } @@ -271,12 +245,6 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { if err != nil { log.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", s.VolID, s.FsName, err) - if errors.Is(err, rados.ErrNotFound) { - // Reset the subVolumeGroupsCreated so that we can try again to create the - // subvolumegroup in next request if the error is Not Found. - s.updateSubVolumeGroupCreated(false) - } - return err }