diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 76de0dfb873..16f713a16ea 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -11,5 +11,6 @@ value [PR](https://github.com/ceph/ceph-csi/pull/4887) - cephfs: support omap data store in radosnamespace [PR](https://github.com/ceph/ceph-csi/pull/4661) - helm: Support setting nodepluigin and provisioner annotations +- rbd: add additional space for encrypted volumes for Luks2 header in [PR](https://github.com/ceph/ceph-csi/pull/4582) ## NOTE diff --git a/e2e/rbd.go b/e2e/rbd.go index fddba2937d5..90a6412acb1 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -24,13 +24,16 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/cryptsetup" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/cloud-provider/volume/helpers" "k8s.io/kubernetes/test/e2e/framework" e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -1998,6 +2001,214 @@ var _ = Describe("RBD", func() { } }) + By("create/resize/clone/restore a encrypted block pvc and verify the image size", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"encrypted": "true", "encryptionType": util.EncryptionTypeBlock.String()}, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + + err = createRBDSnapshotClass(f) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + framework.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + var ( + imageSize uint64 + resizeImageSize uint64 + sizeInBytes int64 + ) + + //nolint:goconst // The string "1Gi" is used multiple times in rbd.go, so it's not a const value. + pvcSize := "1Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(pvcSize)); err != nil { + framework.Failf("failed to parse pvc size: %v", err) + } + imageSize = uint64(sizeInBytes) + cryptsetup.Luks2HeaderSize + + pvc, err := loadPVC(rawPvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + + app, err := loadApp(rawAppPath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + labelKey := "app" + labelValue := "rbd-pod-block-encrypted" + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValue), + } + + app.Labels = map[string]string{labelKey: labelValue} + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate created backend rbd images + err = validateImageSize(f, pvc, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(app, f, &opt, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // create clone PVC and validate the image size + labelValueClonePod := "rbd-pod-block-encrypted-clone" + optClonePod := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueClonePod), + } + + pvcClone, err := loadPVC(pvcBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcClone.Namespace = f.UniqueName + + appClone, err := loadApp(appBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Labels = map[string]string{labelKey: labelValueClonePod} + + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + framework.Failf("failed to create clone PVC and application : %v", err) + } + + err = validateImageSize(f, pvcClone, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(appClone, f, &optClonePod, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // create snapshot and restore PVC and validate the image size + labelValueRestorePod := "rbd-pod-block-encrypted-restore" + optRestorePod := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueRestorePod), + } + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + + err = createSnapshot(&snap, deployTimeout) + if err != nil { + framework.Failf("failed to create snapshot: %v", err) + } + + pvcRestore, err := loadPVC(pvcBlockRestorePath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvcRestore.Spec.DataSource.Name = snap.Name + pvcRestore.Spec.VolumeMode = pvc.Spec.VolumeMode + pvcRestore.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcRestore.Namespace = f.UniqueName + + appRestore, err := loadApp(appBlockRestorePath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + appRestore.Namespace = f.UniqueName + appRestore.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcRestore.Name + appRestore.Labels = map[string]string{labelKey: labelValueRestorePod} + + err = createPVCAndApp("", f, pvcRestore, appRestore, deployTimeout) + if err != nil { + framework.Failf("failed to create clone PVC and application : %v", err) + } + + err = validateImageSize(f, pvcRestore, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(appRestore, f, &optRestorePod, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // resize PVC and validate the image size + resizePVCSize := "2Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(resizePVCSize)); err != nil { + framework.Failf("failed to parse resize pvc size: %v", err) + } + resizeImageSize = uint64(sizeInBytes) + cryptsetup.Luks2HeaderSize + + err = expandPVCSize(f.ClientSet, pvc, resizePVCSize, deployTimeout) + if err != nil { + framework.Failf("failed to expand pvc size: %v", err) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) + if err != nil { + framework.Failf("timeout waiting for pod to be in running state: %v", err) + } + + err = validateImageSize(f, pvc, resizeImageSize) + if err != nil { + framework.Failf("failed to validate image size after resize: %v", err) + } + err = checkDeviceSize(app, f, &opt, resizePVCSize) + if err != nil { + framework.Failf("failed to validate device size after resize: %v", err) + } + + // delete resources + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + framework.Failf("failed to delete pvc and app: %v", err) + } + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + framework.Failf("failed to delete clone pvc and app: %v", err) + } + err = deletePVCAndApp("", f, pvcRestore, appRestore) + if err != nil { + framework.Failf("failed to delete clone pvc and app: %v", err) + } + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + framework.Failf("failed to delete snapshot: %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType) + }) + ByFileAndBlockEncryption("create a PVC and bind it to an app using rbd-nbd mounter with encryption", func( validator encryptionValidateFunc, _ validateFunc, encType util.EncryptionType, ) { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index c2ef08b67d4..deddb2ec2a8 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -1089,6 +1089,7 @@ type imageInfo struct { StripeUnit int `json:"stripe_unit"` StripeCount int `json:"stripe_count"` ObjectSize int `json:"object_size"` + Size uint64 `json:"size"` } // getImageInfo queries rbd about the given image and returns its metadata, and returns @@ -1166,3 +1167,28 @@ func validateStripe(f *framework.Framework, return nil } + +// validateImageSize validates the size of the image. +func validateImageSize(f *framework.Framework, pvc *v1.PersistentVolumeClaim, imageSize uint64) error { + var imgInfo imageInfo + imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) + if err != nil { + return err + } + + imgInfoStr, err := getImageInfo(f, imageData.imageName, defaultRBDPool) + if err != nil { + return err + } + + err = json.Unmarshal([]byte(imgInfoStr), &imgInfo) + if err != nil { + return fmt.Errorf("unmarshal failed: %w. raw buffer response: %s", err, imgInfoStr) + } + + if imgInfo.Size != imageSize { + return fmt.Errorf("image %s size %d does not match expected %d", imgInfo.Name, imgInfo.Size, imageSize) + } + + return nil +} diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 8283970f563..1802693f412 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1226,6 +1226,17 @@ func (cs *ControllerServer) CreateSnapshot( return nil, status.Error(codes.Internal, err.Error()) } + err = vol.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer vol.Destroy(ctx) + + err = vol.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := vol.toSnapshot().ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -1285,6 +1296,17 @@ func cloneFromSnapshot( } } + err = rbdSnap.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer rbdSnap.Destroy(ctx) + + err = rbdSnap.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := rbdSnap.ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index eadef94fbbb..f8e78d73f68 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -60,6 +60,9 @@ const ( metadataDEK = "rbd.csi.ceph.com/dek" oldMetadataDEK = ".rbd.csi.ceph.com/dek" + // luks2 header size metadata key. + luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize" + encryptionPassphraseSize = 20 // rbdDefaultEncryptionType is the default to use when the @@ -131,6 +134,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error { return err } + err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(cryptsetup.Luks2HeaderSize, 10)) + if err != nil { + return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err) + } + err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared) if err != nil { log.ErrorLog(ctx, "failed to save encryption status, deleting "+ diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 274981bfe86..c7d31c3ede7 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -30,6 +30,7 @@ import ( "github.com/ceph/ceph-csi/internal/rbd/types" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/cryptsetup" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" @@ -453,8 +454,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to get IOContext: %w", err) } - err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, - uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options) + size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB) + if pOpts.isBlockEncrypted() { + // When a block-mode PVC is created with encryption enabled, + // some space is reserved for the LUKS2 header. + // Add the LUKS2 header size to the image size so that the user has at least + // the requested size. + size += cryptsetup.Luks2HeaderSize + } + + err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options) if err != nil { return fmt.Errorf("failed to create rbd image: %w", err) } @@ -1643,6 +1652,26 @@ func (ri *rbdImage) GetCreationTime(ctx context.Context) (*time.Time, error) { return ri.CreatedAt, nil } +// getLuks2HeaderSizeSet returns the value of the LUKS2 header size +// set in the image metadata (size returned in MiB). +func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) { + value, err := ri.GetMetadata(luks2HeaderSizeKey) + if err != nil { + if !errors.Is(err, librbd.ErrNotFound) { + return 0, err + } + + return 0, nil + } + + headerSize, parseErr := strconv.ParseUint(value, 10, 64) + if parseErr != nil { + return 0, parseErr + } + + return headerSize, nil +} + // getImageInfo queries rbd about the given image and returns its metadata, and returns // ErrImageNotFound if provided image is not found. func (ri *rbdImage) getImageInfo() error { @@ -1659,6 +1688,14 @@ func (ri *rbdImage) getImageInfo() error { // TODO: can rv.VolSize not be a uint64? Or initialize it to -1? ri.VolSize = int64(imageInfo.Size) + // If the luks2HeaderSizeKey metadata is set + // reduce the extra size of the LUKS header from the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + ri.VolSize -= int64(headerSize) + features, err := image.GetFeatures() if err != nil { return err @@ -1908,7 +1945,17 @@ func (ri *rbdImage) resize(newSize int64) error { } defer image.Close() - err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) + size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB) + + // If the luks2HeaderSizeKey metadata is set + // add the extra size of the LUKS header to the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + size += headerSize + + err = image.Resize(size) if err != nil { return err } diff --git a/internal/util/cryptsetup/cryptsetup.go b/internal/util/cryptsetup/cryptsetup.go index 7423fb417a8..9f58f1b55f9 100644 --- a/internal/util/cryptsetup/cryptsetup.go +++ b/internal/util/cryptsetup/cryptsetup.go @@ -30,6 +30,8 @@ import ( "github.com/ceph/ceph-csi/internal/util/file" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/ceph-csi/internal/util/stripsecrets" + + "k8s.io/cloud-provider/volume/helpers" ) const ( @@ -37,7 +39,10 @@ const ( ExecutionTimeout = 2*time.Minute + 30*time.Second // Limit memory used by Argon2i PBKDF to 32 MiB. - pkdbfMemoryLimit = 32 << 10 // 32768 KiB + cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB + luks2MetadataSize = 32 << 7 // 4096 KiB + luks2KeySlotsSize = 32 << 8 // 8192 KiB + Luks2HeaderSize = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB)) ) // LuksWrapper is a struct that provides a context-aware wrapper around cryptsetup commands. @@ -74,8 +79,12 @@ func (l *luksWrapper) Format(devicePath, passphrase string) (string, string, err "luks2", "--hash", "sha256", + "--luks2-metadata-size", + strconv.Itoa(luks2MetadataSize)+"k", + "--luks2-keyslots-size", + strconv.Itoa(luks2KeySlotsSize)+"k", "--pbkdf-memory", - strconv.Itoa(pkdbfMemoryLimit), + strconv.Itoa(cryptsetupPBKDFMemoryLimit), devicePath, "-d", "/dev/stdin")