diff --git a/internal/journal/snapshotgroupjournal.go b/internal/journal/snapshotgroupjournal.go new file mode 100644 index 000000000000..8a2ad63e7e31 --- /dev/null +++ b/internal/journal/snapshotgroupjournal.go @@ -0,0 +1,417 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package journal + +import ( + "context" + "errors" + "fmt" + + "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" + + "github.com/google/uuid" +) + +const ( + defaultSnapshotGroupNamingPrefix string = "csi-snap-group-" +) + +type SnapshotGroupJournal interface { + setNamespace(ns string) + CheckReservation( + ctx context.Context, + journalPool, + reqName, + namePrefix string) (*SnapshotGroupData, error) + UndoReservation( + ctx context.Context, + csiJournalPool, + snapshotGroupName, + reqName string) error + GetGroupAttributes( + ctx context.Context, + pool, + objectUUID string) (*GroupSnapshotAttributes, error) + ReserveName( + ctx context.Context, + journalPool string, + journalPoolID int64, + reqName, + namePrefix string) (string, string, error) + AddSnapshotVolumeMapping( + ctx context.Context, + pool, + reservedUUID, + snapshotID, + volumeID string) error + RemoveSnapshotVolumeMapping( + ctx context.Context, + pool, + reservedUUID, + snapshotID string) error +} + +type SnapshotGroupJournalConfig struct { + *Config + *Connection +} + +// NewCSISnapshotGroupJournal returns an instance of SnapshotGroupJournal for snapshot groups. +func NewCSISnapshotGroupJournal(suffix string) SnapshotGroupJournal { + return &SnapshotGroupJournalConfig{ + Config: &Config{ + csiDirectory: "csi.snaps.groups." + suffix, + csiNameKeyPrefix: "csi.snap.group", + cephUUIDDirectoryPrefix: "csi.snap.group.", + csiNameKey: "csi.groupname", + namespace: "", + }, + } +} + +// SetNamespace sets the namespace for the journal. +func (sgj *SnapshotGroupJournalConfig) setNamespace(ns string) { + sgj.Config.namespace = ns +} + +// NewCSISnapshotGroupJournalWithNamespace returns an instance of SnapshotGroupJournal for +// snapshot groups using a predetermined namespace value. +func NewCSISnapshotGroupJournalWithNamespace(suffix, ns string) SnapshotGroupJournal { + j := NewCSISnapshotGroupJournal(suffix) + j.setNamespace(ns) + + return j +} + +// Connect establishes a new connection to a ceph cluster for journal metadata. +func (sgj *SnapshotGroupJournalConfig) Connect(monitors, namespace string, cr *util.Credentials) error { + conn, err := sgj.Config.Connect(monitors, namespace, cr) + if err != nil { + return err + } + sgj.Connection = conn + + return nil +} + +// Destroy frees any resources and invalidates the journal connection. +func (sgj *SnapshotGroupJournalConfig) Destroy() { + sgj.Connection.Destroy() +} + +// SnapshotGroupData contains the GroupUUID and GroupSnapshotAttributes for a +// snapshot group. +type SnapshotGroupData struct { + GroupUUID string + GroupSnapshotAttributes *GroupSnapshotAttributes +} + +func generateSnapshotGroupName(namePrefix, groupUUID string) string { + if namePrefix == "" { + namePrefix = defaultSnapshotGroupNamingPrefix + } + + return namePrefix + groupUUID +} + +/* +CheckReservation checks if given request name contains a valid reservation + - If there is a valid reservation, then the corresponding SnapshotGroupData for + the snapshot group is returned + - If there is a reservation that is stale (or not fully cleaned up), it is + garbage collected using the UndoReservation call, as appropriate + +NOTE: As the function manipulates omaps, it should be called with a lock +against the request name held, to prevent parallel operations from modifying +the state of the omaps for this request name. + +Return values: + - SnapshotGroupData: which contains the GroupUUID and GroupSnapshotAttributes + that were reserved for the passed in reqName, empty if there was no + reservation found. + - error: non-nil in case of any errors. +*/ +func (sgj *SnapshotGroupJournalConfig) CheckReservation(ctx context.Context, + journalPool, reqName, namePrefix string, +) (*SnapshotGroupData, error) { + var ( + cj = sgj.Config + snapGroupData = &SnapshotGroupData{} + ) + + // check if request name is already part of the directory omap + fetchKeys := []string{ + cj.csiNameKeyPrefix + reqName, + } + values, err := getOMapValues( + ctx, sgj.Connection, journalPool, cj.namespace, cj.csiDirectory, + cj.commonPrefix, fetchKeys) + if err != nil { + if errors.Is(err, util.ErrKeyNotFound) || errors.Is(err, util.ErrPoolNotFound) { + // pool or omap (oid) was not present + // stop processing but without an error for no reservation exists + return nil, nil + } + + return nil, err + } + objUUID, found := values[cj.csiNameKeyPrefix+reqName] + if !found { + // omap was read but was missing the desired key-value pair + // stop processing but without an error for no reservation exists + return nil, nil + } + snapGroupData.GroupUUID = objUUID + + savedSnapshotGroupAttributes, err := sgj.GetGroupAttributes(ctx, journalPool, + objUUID) + if err != nil { + // error should specifically be not found, for image to be absent, any other error + // is not conclusive, and we should not proceed + if errors.Is(err, util.ErrKeyNotFound) { + err = sgj.UndoReservation(ctx, journalPool, + generateSnapshotGroupName(namePrefix, objUUID), reqName) + } + + return nil, err + } + + // check if the request name in the omap matches the passed in request name + if savedSnapshotGroupAttributes.RequestName != reqName { + // NOTE: This should never be possible, hence no cleanup, but log error + // and return, as cleanup may need to occur manually! + return nil, fmt.Errorf("internal state inconsistent, omap names mismatch,"+ + " request name (%s) snapshot group UUID (%s) snapshot group omap name (%s)", + reqName, objUUID, savedSnapshotGroupAttributes.RequestName) + } + + snapGroupData.GroupSnapshotAttributes.RequestName = savedSnapshotGroupAttributes.RequestName + snapGroupData.GroupSnapshotAttributes.SnaphotVolumeMap = savedSnapshotGroupAttributes.SnaphotVolumeMap + + return snapGroupData, nil +} + +/* +UndoReservation undoes a reservation, in the reverse order of ReserveName +- The UUID directory is cleaned up before the GroupName key in the csiDirectory is cleaned up + +NOTE: Ensure that the Ceph volume snapshots backing the reservation is cleaned up +prior to cleaning up the reservation + +NOTE: As the function manipulates omaps, it should be called with a lock against the request name +held, to prevent parallel operations from modifying the state of the omaps for this request name. + +Input arguments: + - csiJournalPool: Pool name that holds the CSI request name based journal + - snapshotGroupID: ID of the snapshot group, generated from the UUID + - reqName: Request name for the snapshot group +*/ +func (sgj *SnapshotGroupJournalConfig) UndoReservation(ctx context.Context, + csiJournalPool, snapshotGroupID, reqName string, +) error { + // delete volume UUID omap (first, inverse of create order) + + cj := sgj.Config + if snapshotGroupID != "" { + if len(snapshotGroupID) < uuidEncodedLength { + return fmt.Errorf("unable to parse UUID from %s, too short", snapshotGroupID) + } + + snapshotGroupUUID := snapshotGroupID[len(snapshotGroupID)-36:] + if _, err := uuid.Parse(snapshotGroupID); err != nil { + return fmt.Errorf("failed parsing UUID in %s: %w", snapshotGroupID, err) + } + + err := util.RemoveObject( + ctx, + sgj.Connection.monitors, + sgj.Connection.cr, + csiJournalPool, + cj.namespace, + cj.cephUUIDDirectoryPrefix+snapshotGroupUUID) + if err != nil { + if !errors.Is(err, util.ErrObjectNotFound) { + log.ErrorLog(ctx, "failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+snapshotGroupUUID, err) + + return err + } + } + } + + // delete the request name key (last, inverse of create order) + err := removeMapKeys(ctx, sgj.Connection, csiJournalPool, cj.namespace, cj.csiDirectory, + []string{cj.csiNameKeyPrefix + reqName}) + if err != nil { + log.ErrorLog(ctx, "failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err) + } + + return err +} + +/* +ReserveName adds respective entries to the csiDirectory omaps, post generating a target +UUIDDirectory for use. Further, these functions update the UUIDDirectory omaps, to store back +pointers to the CSI generated request names. + +NOTE: As the function manipulates omaps, it should be called with a lock against the request name +held, to prevent parallel operations from modifying the state of the omaps for this request name. + +Input arguments: + - journalPool: Pool where the CSI journal is stored + - journalPoolID: pool ID of the journalPool + - reqName: Name of the volumeSnapshotGroup request received + - namePrefix: Prefix to use when generating the groupSnapshotName name (suffix is an auto-generated UUID) + +Return values: + - string: Contains the UUID that was reserved for the passed in reqName + - string: Contains the VolumeSnapshotGroup name that was reserved for the passed in reqName + - error: non-nil in case of any errors +*/ +func (sgj *SnapshotGroupJournalConfig) ReserveName(ctx context.Context, + journalPool string, journalPoolID int64, + reqName, namePrefix string, +) (string, string, error) { + cj := sgj.Config + + // Create the UUID based omap first, to reserve the same and avoid conflicts + // NOTE: If any service loss occurs post creation of the UUID directory, and before + // setting the request name key to point back to the UUID directory, the + // UUID directory key will be leaked + objUUID, err := reserveOMapName( + ctx, + sgj.Connection.monitors, + sgj.Connection.cr, + journalPool, + cj.namespace, + cj.cephUUIDDirectoryPrefix, + "") + if err != nil { + return "", "", err + } + + groupName := generateSnapshotGroupName(namePrefix, objUUID) + nameKeyVal := objUUID + + // After generating the UUID Directory omap, we populate the csiDirectory + // omap with a key-value entry to map the request to the backend snapshotgroup: + // `csiNameKeyPrefix + reqName: nameKeyVal` + err = setOMapKeys(ctx, sgj.Connection, journalPool, cj.namespace, cj.csiDirectory, + map[string]string{cj.csiNameKeyPrefix + reqName: nameKeyVal}) + if err != nil { + return "", "", err + } + defer func() { + if err != nil { + log.WarningLog(ctx, "reservation failed for snapshotgroup: %s", reqName) + errDefer := sgj.UndoReservation(ctx, journalPool, groupName, reqName) + if errDefer != nil { + log.WarningLog(ctx, "failed undoing reservation of snapshotgroup: %s (%v)", reqName, errDefer) + } + } + }() + + oid := cj.cephUUIDDirectoryPrefix + objUUID + omapValues := map[string]string{} + + // Update UUID directory to store CSI request name + omapValues[cj.csiNameKey] = reqName + + err = setOMapKeys(ctx, sgj.Connection, journalPool, cj.namespace, oid, omapValues) + if err != nil { + return "", "", err + } + + return objUUID, groupName, nil +} + +// GroupSnapshotAttributes contains the request name and the snapshotID's and +// the corresponding volumeID's. +type GroupSnapshotAttributes struct { + RequestName string // Contains the request name for the passed in UUID + SnaphotVolumeMap map[string]string // Contains the snapshotID and the corresponding volumeID mapping +} + +// GetGroupAttributes fetches all keys and their values, from a UUID directory, +// returning GroupSnapshotAttributes structure. +func (sgj *SnapshotGroupJournalConfig) GetGroupAttributes( + ctx context.Context, + pool, objectUUID string, +) (*GroupSnapshotAttributes, error) { + var ( + err error + groupAttributes = &GroupSnapshotAttributes{} + cj = sgj.Config + ) + + values, err := listOMapValues( + ctx, sgj.Connection, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+objectUUID, + cj.commonPrefix) + if err != nil { + if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) { + return nil, err + } + log.WarningLog(ctx, "unable to read omap values: pool missing: %v", err) + } + + groupAttributes.RequestName = values[cj.csiNameKey] + + // Remove request name key from the omap, as we are looking for + // snapshotID/volumeID mapping + delete(values, cj.csiNameKey) + for k, v := range values { + if k == cj.csiImageKey { + groupAttributes.SnaphotVolumeMap = map[string]string{} + groupAttributes.SnaphotVolumeMap[k] = v + } + } + + return groupAttributes, nil +} + +// AddSnapshotVolumeMapping adds a snapshotID and volumeID mapping to the UUID directory. +func (sgj *SnapshotGroupJournalConfig) AddSnapshotVolumeMapping( + ctx context.Context, + pool, + reservedUUID, + snapshotID, + volumeID string, +) error { + err := setOMapKeys(ctx, sgj.Connection, pool, sgj.Config.namespace, sgj.Config.cephUUIDDirectoryPrefix+reservedUUID, + map[string]string{snapshotID: volumeID}) + if err != nil { + return err + } + + return nil +} + +// RemoveSnapshotVolumeMapping removes a snapshotID and volumeID mapping from the UUID directory. +func (sgj *SnapshotGroupJournalConfig) RemoveSnapshotVolumeMapping( + ctx context.Context, + pool, + reservedUUID, + snapshotID string, +) error { + err := removeMapKeys(ctx, sgj.Connection, pool, sgj.Config.namespace, sgj.Config.cephUUIDDirectoryPrefix+reservedUUID, + []string{snapshotID}) + if err != nil { + return err + } + + return nil +}