From fd4e89ed004357c97e7c528af72c6f3e7010c713 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Wed, 5 Jun 2024 11:07:36 +0530 Subject: [PATCH] cephs: support omap store in radosnamespace This commit adds the support for storing the CephFS omap data in a namespace specified in the ceph-csi-config ConfigMap under cephFS.radosNamespace field. If the radosNamespace is not set, the default radosNamespace will be used i.e, csi. Signed-off-by: Praveen M --- internal/cephfs/core/volume.go | 1 + internal/cephfs/groupcontrollerserver.go | 5 ++--- internal/cephfs/store/backingsnapshot.go | 9 ++++----- internal/cephfs/store/fsjournal.go | 19 ++++++------------- internal/cephfs/store/volumegroup.go | 9 ++++----- internal/cephfs/store/volumeoptions.go | 21 ++++++++++++++++++--- 6 files changed, 35 insertions(+), 29 deletions(-) diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index 3e4a9c4892ca..4f8203c7b7c6 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -101,6 +101,7 @@ type SubVolume struct { VolID string // subvolume id. FsName string // filesystem name. SubvolumeGroup string // subvolume group name where subvolume will be created. + RadosNamespace string // rados namespace where omap data will be stored. Pool string // pool name where subvolume will be created. Features []string // subvolume features. Size int64 // subvolume size. diff --git a/internal/cephfs/groupcontrollerserver.go b/internal/cephfs/groupcontrollerserver.go index 321ec5752e06..24b9407163b1 100644 --- a/internal/cephfs/groupcontrollerserver.go +++ b/internal/cephfs/groupcontrollerserver.go @@ -27,7 +27,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" "github.com/ceph/ceph-csi/internal/cephfs/store" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -455,7 +454,7 @@ func (cs *ControllerServer) createSnapshotAndAddMapping( return nil, err } - j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, fsutil.RadosNamespace, cr) + j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, vgo.RadosNamespace, cr) if err != nil { return nil, err } @@ -635,7 +634,7 @@ func (cs *ControllerServer) deleteSnapshotsAndUndoReservation(ctx context.Contex return err } - j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, fsutil.RadosNamespace, cr) + j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, vgo.RadosNamespace, cr) if err != nil { return err } diff --git a/internal/cephfs/store/backingsnapshot.go b/internal/cephfs/store/backingsnapshot.go index 4bd876dcb6ae..43846a1445de 100644 --- a/internal/cephfs/store/backingsnapshot.go +++ b/internal/cephfs/store/backingsnapshot.go @@ -19,7 +19,6 @@ package store import ( "context" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/ceph-csi/internal/util/reftracker" "github.com/ceph/ceph-csi/internal/util/reftracker/radoswrapper" @@ -45,7 +44,7 @@ func AddSnapshotBackedVolumeRef( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(volOptions.RadosNamespace) var ( backingSnapID = volOptions.BackingSnapshotID @@ -90,7 +89,7 @@ func AddSnapshotBackedVolumeRef( if created && !deleted { log.ErrorLog(ctx, "orphaned reftracker object %s (pool %s, namespace %s)", - backingSnapID, volOptions.MetadataPool, fsutil.RadosNamespace) + backingSnapID, volOptions.MetadataPool, volOptions.RadosNamespace) } }() @@ -118,7 +117,7 @@ func UnrefSnapshotBackedVolume( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(volOptions.RadosNamespace) var ( backingSnapID = volOptions.BackingSnapshotID @@ -159,7 +158,7 @@ func UnrefSelfInSnapshotBackedVolumes( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(snapParentVolOptions.RadosNamespace) return reftracker.Remove( radoswrapper.NewIOContext(ioctx), diff --git a/internal/cephfs/store/fsjournal.go b/internal/cephfs/store/fsjournal.go index c9f9a16d7e96..3e3c676e5dfc 100644 --- a/internal/cephfs/store/fsjournal.go +++ b/internal/cephfs/store/fsjournal.go @@ -23,7 +23,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -87,8 +86,7 @@ func CheckVolExists(ctx context.Context, setMetadata bool, ) (*VolumeIdentifier, error) { var vid VolumeIdentifier - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -228,8 +226,7 @@ func UndoVolReservation( } defer cr.DeleteCredentials() - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } @@ -283,8 +280,7 @@ func ReserveVol(ctx context.Context, volOptions *VolumeOptions, secret map[strin return nil, err } - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -329,8 +325,7 @@ func ReserveSnap( err error ) - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -368,8 +363,7 @@ func UndoSnapReservation( snapName string, cr *util.Credentials, ) error { - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } @@ -403,8 +397,7 @@ func CheckSnapExists( setMetadata bool, cr *util.Credentials, ) (*SnapshotIdentifier, error) { - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } diff --git a/internal/cephfs/store/volumegroup.go b/internal/cephfs/store/volumegroup.go index 4286ad76facb..9956293b1812 100644 --- a/internal/cephfs/store/volumegroup.go +++ b/internal/cephfs/store/volumegroup.go @@ -22,7 +22,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -154,7 +153,7 @@ func NewVolumeGroupOptionsFromID( return nil, nil, err } - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, nil, err } @@ -190,7 +189,7 @@ func CheckVolumeGroupSnapExists( cr *util.Credentials, ) (*VolumeGroupSnapshotIdentifier, error) { // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -238,7 +237,7 @@ func ReserveVolumeGroup( vgsi.RequestName = volOptions.RequestName // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -272,7 +271,7 @@ func UndoVolumeGroupReservation( cr *util.Credentials, ) error { // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } diff --git a/internal/cephfs/store/volumeoptions.go b/internal/cephfs/store/volumeoptions.go index 6ec246513c1c..be78dcbbb4ef 100644 --- a/internal/cephfs/store/volumeoptions.go +++ b/internal/cephfs/store/volumeoptions.go @@ -314,6 +314,11 @@ func NewVolumeOptions( return nil, err } + opts.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, opts.ClusterID) + if err != nil { + return nil, err + } + // store topology information from the request opts.TopologyPools, opts.TopologyRequirement, err = util.GetTopologyFromRequest(req) if err != nil { @@ -405,6 +410,10 @@ func NewVolumeOptionsFromVolID( return nil, nil, fmt.Errorf("failed to fetch subvolumegroup list using clusterID (%s): %w", vi.ClusterID, err) } + if volOptions.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, vi.ClusterID); err != nil { + return nil, nil, fmt.Errorf("failed to fetch rados namespace using clusterID (%s): %w", vi.ClusterID, err) + } + cr, err := util.NewAdminCredentials(secrets) if err != nil { return nil, nil, err @@ -435,7 +444,7 @@ func NewVolumeOptionsFromVolID( } // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, nil, err } @@ -788,6 +797,13 @@ func NewSnapshotOptionsFromID( err) } + if volOptions.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, vi.ClusterID); err != nil { + return &volOptions, nil, &sid, fmt.Errorf( + "failed to fetch rados namespace using clusterID (%s): %w", + vi.ClusterID, + err) + } + err = volOptions.Connect(cr) if err != nil { return &volOptions, nil, &sid, err @@ -812,8 +828,7 @@ func NewSnapshotOptionsFromID( return &volOptions, nil, &sid, err } - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return &volOptions, nil, &sid, err }