From 4e8bbf012679978b1e8468230e843e1ad446ce62 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sun, 10 Mar 2019 12:19:02 +0100 Subject: [PATCH 1/7] Supply credentials using volume secrets instead of cli config --- cmd/s3driver/main.go | 21 ++---------- go.mod | 2 ++ go.sum | 10 ++++++ pkg/s3/controllerserver.go | 31 ++++++++++++------ pkg/s3/identityserver.go | 3 +- pkg/s3/mounter.go | 5 ++- pkg/s3/mounter_goofys.go | 4 --- pkg/s3/mounter_rclone.go | 4 --- pkg/s3/mounter_s3backer.go | 11 ++----- pkg/s3/mounter_s3fs.go | 4 --- pkg/s3/mounter_s3ql.go | 4 --- pkg/s3/nodeserver.go | 43 +++++++++--------------- pkg/s3/s3-client.go | 12 +++++++ pkg/s3/s3-driver.go | 16 ++------- pkg/s3/s3-driver_suite_test.go | 60 ++++++++++++++-------------------- pkg/s3/util.go | 20 +++++------- test/secret.yaml | 30 +++++++++++++++++ test/test.sh | 2 +- 18 files changed, 136 insertions(+), 146 deletions(-) create mode 100644 test/secret.yaml diff --git a/cmd/s3driver/main.go b/cmd/s3driver/main.go index a57ac78..0426990 100644 --- a/cmd/s3driver/main.go +++ b/cmd/s3driver/main.go @@ -29,29 +29,14 @@ func init() { } var ( - endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - nodeID = flag.String("nodeid", "", "node id") - accessKeyID = flag.String("access-key-id", "", "S3 Access Key ID to use") - secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use") - s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use") - region = flag.String("region", "", "S3 Region to use") - mounter = flag.String("mounter", "s3fs", "Specify which Mounter to use") - encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)") + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + nodeID = flag.String("nodeid", "", "node id") ) func main() { flag.Parse() - cfg := &s3.Config{ - AccessKeyID: *accessKeyID, - SecretAccessKey: *secretAccessKey, - Endpoint: *s3endpoint, - Region: *region, - Mounter: *mounter, - EncryptionKey: *encryptionKey, - } - - driver, err := s3.NewS3(*nodeID, *endpoint, cfg) + driver, err := s3.NewS3(*nodeID, *endpoint) if err != nil { log.Fatal(err) } diff --git a/go.mod b/go.mod index 0283831..6fe64b0 100644 --- a/go.mod +++ b/go.mod @@ -42,3 +42,5 @@ require ( k8s.io/kubernetes v1.13.4 k8s.io/utils v0.0.0-20180703210027-ab9069044f32 // indirect ) + +replace github.com/kubernetes-csi/csi-test => github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f diff --git a/go.sum b/go.sum index 38372b6..74789b9 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,14 @@ github.com/aws/aws-sdk-go v1.14.27 h1:fRVME5X3sxZnctdCcabNTWZq7ZGrpVgUAYk4OA5EG0 github.com/aws/aws-sdk-go v1.14.27/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E= github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/ctrox/csi-test v1.1.0 h1:YwOvPrlZw6/qgG+G8EQMkMniPt2HJmTOYVBiawgfiQ8= +github.com/ctrox/csi-test v1.1.0/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= +github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f h1:FLD1xv7Vwv7+JZizABfim+tR8Ctj68B2mnS529kHBPg= +github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= +github.com/ctrox/csi-test v1.1.2-0.20190310094942-e965dacfef26 h1:KbZ3qIvoWP0CD7ZnUULipd5QGg0gmNLCfxikgAYnKwQ= +github.com/ctrox/csi-test v1.1.2-0.20190310094942-e965dacfef26/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= +github.com/ctrox/csi-test v1.1.2-0.20190310103005-3f3cc7817699 h1:bQ82DNERrJuin7/+sRCoeBz7FV8/HNS6LpIe48XUWCo= +github.com/ctrox/csi-test v1.1.2-0.20190310103005-3f3cc7817699/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY= @@ -30,6 +38,8 @@ github.com/kahing/goofys v0.19.0 h1:jcuffrnpvZq+LjXtRODo0pvNOglw32ClzBZ1XLShFnk= github.com/kahing/goofys v0.19.0/go.mod h1:erC9E45nY5m8v6FE+tYIGRVjIC2N8viMlJrgrsXB2Q4= github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdTUBXB4xE4= github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +github.com/kubernetes-csi/csi-test v1.1.1 h1:L4RPre34ICeoQW7ez4X5t0PnFKaKs8K5q0c1XOrvXEM= +github.com/kubernetes-csi/csi-test v1.1.1/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e h1:BkkRJIF329ps8digiMWthYzDPl9KB8PwkDwvVWDwM4A= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index dca3b2f..49e8f7b 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -29,12 +29,11 @@ import ( "google.golang.org/grpc/status" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) type controllerServer struct { *csicommon.DefaultControllerServer - *s3 } func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { @@ -59,13 +58,17 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol glog.V(4).Infof("Got a request to create volume %s", volumeID) - exists, err := cs.s3.client.bucketExists(volumeID) + s3, err := newS3ClientFromSecrets(req.GetSecrets()) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 client: %s", err) + } + exists, err := s3.bucketExists(volumeID) if err != nil { return nil, fmt.Errorf("failed to check if bucket %s exists: %v", volumeID, err) } if exists { var b *bucket - b, err = cs.s3.client.getBucket(volumeID) + b, err = s3.getBucket(volumeID) if err != nil { return nil, fmt.Errorf("failed to get bucket metadata of bucket %s: %v", volumeID, err) } @@ -74,10 +77,10 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Volume with the same name: %s but with smaller size already exist", volumeID)) } } else { - if err = cs.s3.client.createBucket(volumeID); err != nil { + if err = s3.createBucket(volumeID); err != nil { return nil, fmt.Errorf("failed to create volume %s: %v", volumeID, err) } - if err = cs.s3.client.createPrefix(volumeID, fsPrefix); err != nil { + if err = s3.createPrefix(volumeID, fsPrefix); err != nil { return nil, fmt.Errorf("failed to create prefix %s: %v", fsPrefix, err) } } @@ -87,7 +90,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol CapacityBytes: capacityBytes, FSPath: fsPrefix, } - if err := cs.s3.client.setBucket(b); err != nil { + if err := s3.setBucket(b); err != nil { return nil, fmt.Errorf("Error setting bucket metadata: %v", err) } @@ -118,12 +121,16 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol } glog.V(4).Infof("Deleting volume %s", volumeID) - exists, err := cs.s3.client.bucketExists(volumeID) + s3, err := newS3ClientFromSecrets(req.GetSecrets()) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 client: %s", err) + } + exists, err := s3.bucketExists(volumeID) if err != nil { return nil, err } if exists { - if err := cs.s3.client.removeBucket(volumeID); err != nil { + if err := s3.removeBucket(volumeID); err != nil { glog.V(3).Infof("Failed to remove volume: %v", err) return nil, err } @@ -144,7 +151,11 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req return nil, status.Error(codes.InvalidArgument, "Volume capabilities missing in request") } - exists, err := cs.s3.client.bucketExists(req.GetVolumeId()) + s3, err := newS3ClientFromSecrets(req.GetSecrets()) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 client: %s", err) + } + exists, err := s3.bucketExists(req.GetVolumeId()) if err != nil { return nil, err } diff --git a/pkg/s3/identityserver.go b/pkg/s3/identityserver.go index 9e46faf..6e04fa8 100644 --- a/pkg/s3/identityserver.go +++ b/pkg/s3/identityserver.go @@ -17,10 +17,9 @@ limitations under the License. package s3 import ( - "github.com/kubernetes-csi/drivers/pkg/csi-common" + csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) type identityServer struct { *csicommon.DefaultIdentityServer - *s3 } diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index 6bddc39..df17c7a 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -15,7 +15,6 @@ type Mounter interface { Stage(stagePath string) error Unstage(stagePath string) error Mount(source string, target string) error - Unmount(target string) error } const ( @@ -67,12 +66,12 @@ func fuseMount(path string, command string, args []string) error { return waitForMount(path, 10*time.Second) } -func fuseUnmount(path string, command string) error { +func fuseUnmount(path string) error { if err := mount.New("").Unmount(path); err != nil { return err } // as fuse quits immediately, we will try to wait until the process is done - process, err := findFuseMountProcess(path, command) + process, err := findFuseMountProcess(path) if err != nil { glog.Errorf("Error getting PID of fuse mount: %s", err) return nil diff --git a/pkg/s3/mounter_goofys.go b/pkg/s3/mounter_goofys.go index 6e93bb0..98a176b 100644 --- a/pkg/s3/mounter_goofys.go +++ b/pkg/s3/mounter_goofys.go @@ -69,7 +69,3 @@ func (goofys *goofysMounter) Mount(source string, target string) error { } return nil } - -func (goofys *goofysMounter) Unmount(targetPath string) error { - return fuseUnmount(targetPath, goofysCmd) -} diff --git a/pkg/s3/mounter_rclone.go b/pkg/s3/mounter_rclone.go index 60d1203..abc2d32 100644 --- a/pkg/s3/mounter_rclone.go +++ b/pkg/s3/mounter_rclone.go @@ -54,7 +54,3 @@ func (rclone *rcloneMounter) Mount(source string, target string) error { os.Setenv("AWS_SECRET_ACCESS_KEY", rclone.secretAccessKey) return fuseMount(target, rcloneCmd, args) } - -func (rclone *rcloneMounter) Unmount(target string) error { - return fuseUnmount(target, rcloneCmd) -} diff --git a/pkg/s3/mounter_s3backer.go b/pkg/s3/mounter_s3backer.go index 204df6d..118263c 100644 --- a/pkg/s3/mounter_s3backer.go +++ b/pkg/s3/mounter_s3backer.go @@ -71,14 +71,14 @@ func (s3backer *s3backerMounter) Stage(stageTarget string) error { // ensure 'file' device is formatted err := formatFs(s3backerFsType, path.Join(stageTarget, s3backerDevice)) if err != nil { - fuseUnmount(stageTarget, s3backerCmd) + fuseUnmount(stageTarget) } return err } func (s3backer *s3backerMounter) Unstage(stageTarget string) error { // Unmount the s3backer fuse mount - return fuseUnmount(stageTarget, s3backerCmd) + return fuseUnmount(stageTarget) } func (s3backer *s3backerMounter) Mount(source string, target string) error { @@ -87,17 +87,12 @@ func (s3backer *s3backerMounter) Mount(source string, target string) error { err := mount.New("").Mount(device, target, s3backerFsType, []string{}) if err != nil { // cleanup fuse mount - fuseUnmount(target, s3backerCmd) + fuseUnmount(target) return err } return nil } -func (s3backer *s3backerMounter) Unmount(targetPath string) error { - // Unmount the filesystem first - return mount.New("").Unmount(targetPath) -} - func (s3backer *s3backerMounter) mountInit(path string) error { args := []string{ fmt.Sprintf("--blockSize=%s", s3backerBlockSize), diff --git a/pkg/s3/mounter_s3fs.go b/pkg/s3/mounter_s3fs.go index e332557..772d6c0 100644 --- a/pkg/s3/mounter_s3fs.go +++ b/pkg/s3/mounter_s3fs.go @@ -51,10 +51,6 @@ func (s3fs *s3fsMounter) Mount(source string, target string) error { return fuseMount(target, s3fsCmd, args) } -func (s3fs *s3fsMounter) Unmount(target string) error { - return fuseUnmount(target, s3fsCmd) -} - func writes3fsPass(pwFileContent string) error { pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go index 1f90481..d32c47d 100644 --- a/pkg/s3/mounter_s3ql.go +++ b/pkg/s3/mounter_s3ql.go @@ -93,10 +93,6 @@ func (s3ql *s3qlMounter) Mount(source string, target string) error { return fuseMount(target, s3qlCmdMount, append(args, s3ql.options...)) } -func (s3ql *s3qlMounter) Unmount(target string) error { - return fuseUnmount(target, s3qlCmdMount) -} - func (s3ql *s3qlMounter) writeConfig() error { s3qlIni := ini.Empty() section, err := s3qlIni.NewSection("s3ql") diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 36fbd4d..36f985f 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -17,6 +17,7 @@ limitations under the License. package s3 import ( + "fmt" "os" "github.com/golang/glog" @@ -27,12 +28,11 @@ import ( "google.golang.org/grpc/status" "k8s.io/kubernetes/pkg/util/mount" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) type nodeServer struct { *csicommon.DefaultNodeServer - *s3 } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -76,12 +76,16 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - b, err := ns.s3.client.getBucket(volumeID) + s3, err := newS3ClientFromSecrets(req.GetSecrets()) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 client: %s", err) + } + b, err := s3.getBucket(volumeID) if err != nil { return nil, err } - mounter, err := newMounter(b, ns.s3.cfg) + mounter, err := newMounter(b, s3.cfg) if err != nil { return nil, err } @@ -106,15 +110,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.InvalidArgument, "Target path missing in request") } - b, err := ns.s3.client.getBucket(volumeID) - if err != nil { - return nil, err - } - mounter, err := newMounter(b, ns.s3.cfg) - if err != nil { - return nil, err - } - if err := mounter.Unmount(targetPath); err != nil { + if err := fuseUnmount(targetPath); err != nil { return nil, status.Error(codes.Internal, err.Error()) } glog.V(4).Infof("s3: bucket %s has been unmounted.", volumeID) @@ -146,12 +142,15 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol if !notMnt { return &csi.NodeStageVolumeResponse{}, nil } - b, err := ns.s3.client.getBucket(volumeID) + s3, err := newS3ClientFromSecrets(req.GetSecrets()) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 client: %s", err) + } + b, err := s3.getBucket(volumeID) if err != nil { return nil, err } - - mounter, err := newMounter(b, ns.s3.cfg) + mounter, err := newMounter(b, s3.cfg) if err != nil { return nil, err } @@ -174,18 +173,6 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Error(codes.InvalidArgument, "Target path missing in request") } - b, err := ns.s3.client.getBucket(volumeID) - if err != nil { - return nil, err - } - mounter, err := newMounter(b, ns.s3.cfg) - if err != nil { - return nil, err - } - if err := mounter.Unstage(stagingTargetPath); err != nil { - return nil, err - } - return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/pkg/s3/s3-client.go b/pkg/s3/s3-client.go index 8591caf..2c3ef59 100644 --- a/pkg/s3/s3-client.go +++ b/pkg/s3/s3-client.go @@ -49,6 +49,18 @@ func newS3Client(cfg *Config) (*s3Client, error) { return client, nil } +func newS3ClientFromSecrets(secrets map[string]string) (*s3Client, error) { + return newS3Client(&Config{ + AccessKeyID: secrets["accessKeyID"], + SecretAccessKey: secrets["secretAccessKey"], + Region: secrets["region"], + Endpoint: secrets["endpoint"], + EncryptionKey: secrets["encryptionKey"], + // Mounter is set in the volume preferences, not secrets + Mounter: "", + }) +} + func (client *s3Client) bucketExists(bucketName string) (bool, error) { return client.minio.BucketExists(bucketName) } diff --git a/pkg/s3/s3-driver.go b/pkg/s3/s3-driver.go index 522562e..5431829 100644 --- a/pkg/s3/s3-driver.go +++ b/pkg/s3/s3-driver.go @@ -20,14 +20,12 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) type s3 struct { driver *csicommon.CSIDriver - client *s3Client endpoint string - cfg *Config ids *identityServer ns *nodeServer @@ -47,22 +45,15 @@ var ( ) // NewS3 initializes the driver -func NewS3(nodeID string, endpoint string, cfg *Config) (*s3, error) { +func NewS3(nodeID string, endpoint string) (*s3, error) { driver := csicommon.NewCSIDriver(driverName, vendorVersion, nodeID) if driver == nil { glog.Fatalln("Failed to initialize CSI Driver.") } - client, err := newS3Client(cfg) - if err != nil { - glog.V(3).Infof("Failed to create s3 client: %v", err) - return nil, err - } s3Driver := &s3{ endpoint: endpoint, driver: driver, - client: client, - cfg: cfg, } return s3Driver, nil } @@ -70,21 +61,18 @@ func NewS3(nodeID string, endpoint string, cfg *Config) (*s3, error) { func (s3 *s3) newIdentityServer(d *csicommon.CSIDriver) *identityServer { return &identityServer{ DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d), - s3: s3, } } func (s3 *s3) newControllerServer(d *csicommon.CSIDriver) *controllerServer { return &controllerServer{ DefaultControllerServer: csicommon.NewDefaultControllerServer(d), - s3: s3, } } func (s3 *s3) newNodeServer(d *csicommon.CSIDriver) *nodeServer { return &nodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(d), - s3: s3, } } diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go index 8fbe0f7..aa6d809 100644 --- a/pkg/s3/s3-driver_suite_test.go +++ b/pkg/s3/s3-driver_suite_test.go @@ -24,16 +24,10 @@ var _ = Describe("S3Driver", func() { Context("goofys", func() { socket := "/tmp/csi-goofys.sock" csiEndpoint := "unix://" + socket - cfg := &s3.Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - Mounter: "goofys", - } if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { Expect(err).NotTo(HaveOccurred()) } - driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + driver, err := s3.NewS3("test-node", csiEndpoint) if err != nil { log.Fatal(err) } @@ -44,6 +38,10 @@ var _ = Describe("S3Driver", func() { TargetPath: mntDir, StagingPath: stagingDir, Address: csiEndpoint, + SecretsFile: "../../test/secret.yaml", + TestVolumeParameters: map[string]string{ + "mounter": "goofys", + }, } sanity.GinkgoTest(sanityCfg) }) @@ -52,16 +50,10 @@ var _ = Describe("S3Driver", func() { Context("s3fs", func() { socket := "/tmp/csi-s3fs.sock" csiEndpoint := "unix://" + socket - cfg := &s3.Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - Mounter: "s3fs", - } if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { Expect(err).NotTo(HaveOccurred()) } - driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + driver, err := s3.NewS3("test-node", csiEndpoint) if err != nil { log.Fatal(err) } @@ -72,6 +64,10 @@ var _ = Describe("S3Driver", func() { TargetPath: mntDir, StagingPath: stagingDir, Address: csiEndpoint, + SecretsFile: "../../test/secret.yaml", + TestVolumeParameters: map[string]string{ + "mounter": "s3fs", + }, } sanity.GinkgoTest(sanityCfg) }) @@ -81,16 +77,10 @@ var _ = Describe("S3Driver", func() { socket := "/tmp/csi-s3ql.sock" csiEndpoint := "unix://" + socket - cfg := &s3.Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - Mounter: "s3ql", - } if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { Expect(err).NotTo(HaveOccurred()) } - driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + driver, err := s3.NewS3("test-node", csiEndpoint) if err != nil { log.Fatal(err) } @@ -103,6 +93,10 @@ var _ = Describe("S3Driver", func() { TargetPath: mntDir, StagingPath: stagingDir, Address: csiEndpoint, + SecretsFile: "../../test/secret.yaml", + TestVolumeParameters: map[string]string{ + "mounter": "s3ql", + }, } sanity.GinkgoTest(sanityCfg) }) @@ -112,18 +106,12 @@ var _ = Describe("S3Driver", func() { socket := "/tmp/csi-s3backer.sock" csiEndpoint := "unix://" + socket - cfg := &s3.Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - Mounter: "s3backer", - } if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { Expect(err).NotTo(HaveOccurred()) } // Clear loop device so we cover the creation of it os.Remove(s3.S3backerLoopDevice) - driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + driver, err := s3.NewS3("test-node", csiEndpoint) if err != nil { log.Fatal(err) } @@ -134,6 +122,10 @@ var _ = Describe("S3Driver", func() { TargetPath: mntDir, StagingPath: stagingDir, Address: csiEndpoint, + SecretsFile: "../../test/secret.yaml", + TestVolumeParameters: map[string]string{ + "mounter": "s3backer", + }, } sanity.GinkgoTest(sanityCfg) }) @@ -143,16 +135,10 @@ var _ = Describe("S3Driver", func() { socket := "/tmp/csi-rclone.sock" csiEndpoint := "unix://" + socket - cfg := &s3.Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - Mounter: "rclone", - } if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { Expect(err).NotTo(HaveOccurred()) } - driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + driver, err := s3.NewS3("test-node", csiEndpoint) if err != nil { log.Fatal(err) } @@ -163,6 +149,10 @@ var _ = Describe("S3Driver", func() { TargetPath: mntDir, StagingPath: stagingDir, Address: csiEndpoint, + SecretsFile: "../../test/secret.yaml", + TestVolumeParameters: map[string]string{ + "mounter": "rclone", + }, } sanity.GinkgoTest(sanityCfg) }) diff --git a/pkg/s3/util.go b/pkg/s3/util.go index e0bc29f..da95a7c 100644 --- a/pkg/s3/util.go +++ b/pkg/s3/util.go @@ -60,22 +60,20 @@ func waitForMount(path string, timeout time.Duration) error { } } -func findFuseMountProcess(path string, name string) (*os.Process, error) { +func findFuseMountProcess(path string) (*os.Process, error) { processes, err := ps.Processes() if err != nil { return nil, err } for _, p := range processes { - if strings.Contains(p.Executable(), name) { - cmdLine, err := getCmdLine(p.Pid()) - if err != nil { - glog.Errorf("Unable to get cmdline of PID %v: %s", p.Pid(), err) - continue - } - if strings.Contains(cmdLine, path) { - glog.Infof("Found matching pid %v on path %s", p.Pid(), path) - return os.FindProcess(p.Pid()) - } + cmdLine, err := getCmdLine(p.Pid()) + if err != nil { + glog.Errorf("Unable to get cmdline of PID %v: %s", p.Pid(), err) + continue + } + if strings.Contains(cmdLine, path) { + glog.Infof("Found matching pid %v on path %s", p.Pid(), path) + return os.FindProcess(p.Pid()) } } return nil, nil diff --git a/test/secret.yaml b/test/secret.yaml new file mode 100644 index 0000000..1f08c2f --- /dev/null +++ b/test/secret.yaml @@ -0,0 +1,30 @@ +CreateVolumeSecret: + accessKeyID: FJDSJ + secretAccessKey: DSG643HGDS + endpoint: http://127.0.0.1:9000 + region: "" + encryptionKey: "" +DeleteVolumeSecret: + accessKeyID: FJDSJ + secretAccessKey: DSG643HGDS + endpoint: http://127.0.0.1:9000 + region: "" + encryptionKey: "" +NodeStageVolumeSecret: + accessKeyID: FJDSJ + secretAccessKey: DSG643HGDS + endpoint: http://127.0.0.1:9000 + region: "" + encryptionKey: "" +NodePublishVolumeSecret: + accessKeyID: FJDSJ + secretAccessKey: DSG643HGDS + endpoint: http://127.0.0.1:9000 + region: "" + encryptionKey: "" +ControllerValidateVolumeCapabilitiesSecret: + accessKeyID: FJDSJ + secretAccessKey: DSG643HGDS + endpoint: http://127.0.0.1:9000 + region: "" + encryptionKey: "" \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index c5af8b6..0e8b809 100755 --- a/test/test.sh +++ b/test/test.sh @@ -5,4 +5,4 @@ export MINIO_SECRET_KEY=DSG643HGDS mkdir -p /tmp/minio minio server --quiet /tmp/minio &>/dev/null & sleep 5 -go test github.com/ctrox/csi-s3/pkg/s3 -cover +go test github.com/ctrox/csi-s3/pkg/s3 -cover -coverprofile=coverage.out \ No newline at end of file From c6b3b5199d57c4cd94b232dbf48f405bc2a38dbb Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Tue, 14 May 2019 21:01:54 +0200 Subject: [PATCH 2/7] Upgrade csi-spec, fix tests --- go.mod | 18 +++++++----------- go.sum | 26 ++++++++++++++++++++++++++ pkg/s3/controllerserver.go | 6 +++++- pkg/s3/nodeserver.go | 4 ++++ pkg/s3/s3-client.go | 3 ++- pkg/s3/s3-driver_suite_test.go | 32 +++++++++++--------------------- test/Dockerfile | 18 ++++++++++-------- test/test.sh | 4 ++-- 8 files changed, 67 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 6fe64b0..6e4368c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ctrox/csi-s3 require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/aws/aws-sdk-go v1.14.27 // indirect - github.com/container-storage-interface/spec v1.0.0 + github.com/container-storage-interface/spec v1.1.0 github.com/go-ini/ini v1.38.1 // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b @@ -13,11 +13,11 @@ require ( github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect github.com/kahing/go-xattr v1.1.1 // indirect github.com/kahing/goofys v0.19.0 - github.com/kubernetes-csi/csi-test v1.1.0 - github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e + github.com/kubernetes-csi/csi-lib-utils v0.6.1 // indirect + github.com/kubernetes-csi/csi-test v2.0.0+incompatible + github.com/kubernetes-csi/drivers v1.0.2 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect - github.com/minio/minio-go v6.0.5+incompatible - github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9 // indirect + github.com/minio/minio-go v0.0.0-20190430232750-10b3660b8f09 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 github.com/onsi/ginkgo v1.5.0 github.com/onsi/gomega v1.4.0 @@ -27,20 +27,16 @@ require ( github.com/spf13/afero v1.2.1 // indirect github.com/stretchr/testify v1.3.0 // indirect github.com/urfave/cli v1.20.0 // indirect - golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 // indirect - golang.org/x/net v0.0.0-20180712202826-d0887baf81f4 + golang.org/x/net v0.0.0-20190213061140-3a22650c66bd golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect - golang.org/x/sys v0.0.0-20180715085529-ac767d655b30 // indirect google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b // indirect google.golang.org/grpc v1.13.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect - gopkg.in/ini.v1 v1.38.1 + gopkg.in/ini.v1 v1.41.0 gopkg.in/yaml.v2 v2.2.1 // indirect k8s.io/apimachinery v0.0.0-20180714051327-705cfa51a97f // indirect k8s.io/klog v0.2.0 // indirect k8s.io/kubernetes v1.13.4 k8s.io/utils v0.0.0-20180703210027-ab9069044f32 // indirect ) - -replace github.com/kubernetes-csi/csi-test => github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f diff --git a/go.sum b/go.sum index 74789b9..f4a75ff 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,15 @@ github.com/aws/aws-sdk-go v1.14.27 h1:fRVME5X3sxZnctdCcabNTWZq7ZGrpVgUAYk4OA5EG0 github.com/aws/aws-sdk-go v1.14.27/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E= github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.1.0 h1:qPsTqtR1VUPvMPeK0UnCZMtXaKGyyLPG8gj/wG6VqMs= +github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/ctrox/csi-test v0.0.0-20190311173153-80a2484bf798 h1:nfii2XdBGLaje6HWjtMCKaUBRv86HLg9uiOtAW9NRJA= +github.com/ctrox/csi-test v0.0.0-20190311173153-80a2484bf798/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= github.com/ctrox/csi-test v1.1.0 h1:YwOvPrlZw6/qgG+G8EQMkMniPt2HJmTOYVBiawgfiQ8= github.com/ctrox/csi-test v1.1.0/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f h1:FLD1xv7Vwv7+JZizABfim+tR8Ctj68B2mnS529kHBPg= github.com/ctrox/csi-test v1.1.1-0.20190310103436-e50382dcb47f/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= +github.com/ctrox/csi-test v1.1.1-0.20190311173153-80a2484bf798/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= github.com/ctrox/csi-test v1.1.2-0.20190310094942-e965dacfef26 h1:KbZ3qIvoWP0CD7ZnUULipd5QGg0gmNLCfxikgAYnKwQ= github.com/ctrox/csi-test v1.1.2-0.20190310094942-e965dacfef26/go.mod h1:Sdb3sQ5DaEikqpKZNzj+abr8x/OCMXB0KTaxIAXP1RI= github.com/ctrox/csi-test v1.1.2-0.20190310103005-3f3cc7817699 h1:bQ82DNERrJuin7/+sRCoeBz7FV8/HNS6LpIe48XUWCo= @@ -24,6 +29,7 @@ github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb h1:TRAjtOoio6kvnrIMLeXesGT9IydfO+zQoioKWrv40nI= github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbddQ0uScZCHQoZsT5NykGRCeeeIs= @@ -36,18 +42,28 @@ github.com/kahing/go-xattr v1.1.1 h1:7Ft/P9Gc6iqRVzBRLVw/yLL/dbtzL6FsZzGQj3T9ZY8 github.com/kahing/go-xattr v1.1.1/go.mod h1:DXZs3JwPmH2DnyFxWjLZWb65lq8pOPtsf9LD+2Gbbpw= github.com/kahing/goofys v0.19.0 h1:jcuffrnpvZq+LjXtRODo0pvNOglw32ClzBZ1XLShFnk= github.com/kahing/goofys v0.19.0/go.mod h1:erC9E45nY5m8v6FE+tYIGRVjIC2N8viMlJrgrsXB2Q4= +github.com/kubernetes-csi/csi-lib-utils v0.6.1 h1:+AZ58SRSRWh2vmMoWAAGcv7x6fIyBMpyCXAgIc9kT28= +github.com/kubernetes-csi/csi-lib-utils v0.6.1/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ= github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdTUBXB4xE4= github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/csi-test v1.1.1 h1:L4RPre34ICeoQW7ez4X5t0PnFKaKs8K5q0c1XOrvXEM= github.com/kubernetes-csi/csi-test v1.1.1/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +github.com/kubernetes-csi/csi-test v2.0.0+incompatible h1:ia04uVFUM/J9n/v3LEMn3rEG6FmKV5BH9QLw7H68h44= +github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e h1:BkkRJIF329ps8digiMWthYzDPl9KB8PwkDwvVWDwM4A= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY= +github.com/kubernetes-csi/drivers v1.0.2 h1:kaEAMfo+W5YFr23yedBIY+NGnNjr6/PbPzx7N4GYgiQ= +github.com/kubernetes-csi/drivers v1.0.2/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/minio/minio-go v0.0.0-20190430232750-10b3660b8f09 h1:c64QOQYYVNo2a9kaHCgwyUyllGDYZVMcRGwzBUQMUao= +github.com/minio/minio-go v0.0.0-20190430232750-10b3660b8f09/go.mod h1:/haSOWG8hQNx2+JOfLJ9GKp61EAmgPwRVw/Sac0NzaM= github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrMenWrjM6nZh0Q= github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9 h1:Y94YB7jrsihrbGSqRNMwRWJ2/dCxr0hdC2oPRohkx0A= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo= github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/onsi/ginkgo v1.5.0 h1:uZr+v/TFDdYkdA+j02sPO1kA5owrfjBGCJAogfIyThE= @@ -62,6 +78,8 @@ github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= @@ -73,12 +91,18 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 h1:h7zdf0RiEvWbYBKIx4b+q41xoUVnMmvsGZnIVE5syG8= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b h1:Ib/yptP38nXZFMwqWSip+OKuMP9OkyDe3p+DssP8n9w= +golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180712202826-d0887baf81f4 h1:KDF3PK6A+dkI7c4O8QbMtJqcXE3LdNJFGZECIlifQOg= golang.org/x/net v0.0.0-20180712202826-d0887baf81f4/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180715085529-ac767d655b30 h1:4bYUqrXBoiI7UFQeibUwFhvcHfaEeL75O3lOcZa964o= golang.org/x/sys v0.0.0-20180715085529-ac767d655b30/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e h1:3GIlrlVLfkoipSReOMNAgApI0ajnalyLa/EZHHca/XI= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b h1:mXqBiicV0B+k8wzFNkKeNBRL7LyRV5xG0s+S6ffLb/E= @@ -93,6 +117,8 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNj gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/ini.v1 v1.38.1 h1:8E3nEICVJ6kxl6aTXYp77xYyObhw7YG9/avdj0r3vME= gopkg.in/ini.v1 v1.38.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE= +gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/apimachinery v0.0.0-20180714051327-705cfa51a97f h1:mjXiDUfs+4mhzRTLNTkAfQS9lqJCXQN/fIcMysNGW/Y= diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 49e8f7b..1b25c6d 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -131,7 +131,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol } if exists { if err := s3.removeBucket(volumeID); err != nil { - glog.V(3).Infof("Failed to remove volume: %v", err) + glog.V(3).Infof("Failed to remove volume %s: %v", volumeID, err) return nil, err } } else { @@ -186,6 +186,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req }, nil } +func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { + return &csi.ControllerExpandVolumeResponse{}, status.Error(codes.Unimplemented, "ControllerExpandVolume is not implemented") +} + func sanitizeVolumeID(volumeID string) string { volumeID = strings.ToLower(volumeID) if len(volumeID) > 63 { diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 36f985f..5830e53 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -194,6 +194,10 @@ func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, nil } +func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + return &csi.NodeExpandVolumeResponse{}, status.Error(codes.Unimplemented, "NodeExpandVolume is not implemented") +} + func checkMount(targetPath string) (bool, error) { notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) if err != nil { diff --git a/pkg/s3/s3-client.go b/pkg/s3/s3-client.go index 2c3ef59..b85bc67 100644 --- a/pkg/s3/s3-client.go +++ b/pkg/s3/s3-client.go @@ -120,7 +120,8 @@ func (client *s3Client) emptyBucket(bucketName string) error { } } - return nil + // ensure our prefix is also removed + return client.minio.RemoveObject(bucketName, fsPrefix) } func (client *s3Client) setBucket(bucket *bucket) error { diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go index aa6d809..dd98fe5 100644 --- a/pkg/s3/s3-driver_suite_test.go +++ b/pkg/s3/s3-driver_suite_test.go @@ -1,25 +1,17 @@ package s3_test import ( - "io/ioutil" "log" "os" + "github.com/ctrox/csi-s3/pkg/s3" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/ctrox/csi-s3/pkg/s3" "github.com/kubernetes-csi/csi-test/pkg/sanity" ) var _ = Describe("S3Driver", func() { - mntDir, _ := ioutil.TempDir("", "mnt") - stagingDir, _ := ioutil.TempDir("", "staging") - - AfterSuite(func() { - os.RemoveAll(mntDir) - os.RemoveAll(stagingDir) - }) Context("goofys", func() { socket := "/tmp/csi-goofys.sock" @@ -35,8 +27,8 @@ var _ = Describe("S3Driver", func() { Describe("CSI sanity", func() { sanityCfg := &sanity.Config{ - TargetPath: mntDir, - StagingPath: stagingDir, + TargetPath: os.TempDir() + "/goofys-target", + StagingPath: os.TempDir() + "/goofys-staging", Address: csiEndpoint, SecretsFile: "../../test/secret.yaml", TestVolumeParameters: map[string]string{ @@ -61,8 +53,8 @@ var _ = Describe("S3Driver", func() { Describe("CSI sanity", func() { sanityCfg := &sanity.Config{ - TargetPath: mntDir, - StagingPath: stagingDir, + TargetPath: os.TempDir() + "/s3fs-target", + StagingPath: os.TempDir() + "/s3fs-staging", Address: csiEndpoint, SecretsFile: "../../test/secret.yaml", TestVolumeParameters: map[string]string{ @@ -86,12 +78,10 @@ var _ = Describe("S3Driver", func() { } go driver.Run() - defer os.RemoveAll(mntDir) - Describe("CSI sanity", func() { sanityCfg := &sanity.Config{ - TargetPath: mntDir, - StagingPath: stagingDir, + TargetPath: os.TempDir() + "/s3ql-target", + StagingPath: os.TempDir() + "/s3ql-staging", Address: csiEndpoint, SecretsFile: "../../test/secret.yaml", TestVolumeParameters: map[string]string{ @@ -119,8 +109,8 @@ var _ = Describe("S3Driver", func() { Describe("CSI sanity", func() { sanityCfg := &sanity.Config{ - TargetPath: mntDir, - StagingPath: stagingDir, + TargetPath: os.TempDir() + "/s3backer-target", + StagingPath: os.TempDir() + "/s3backer-staging", Address: csiEndpoint, SecretsFile: "../../test/secret.yaml", TestVolumeParameters: map[string]string{ @@ -146,8 +136,8 @@ var _ = Describe("S3Driver", func() { Describe("CSI sanity", func() { sanityCfg := &sanity.Config{ - TargetPath: mntDir, - StagingPath: stagingDir, + TargetPath: os.TempDir() + "/rclone-target", + StagingPath: os.TempDir() + "/rclone-staging", Address: csiEndpoint, SecretsFile: "../../test/secret.yaml", TestVolumeParameters: map[string]string{ diff --git a/test/Dockerfile b/test/Dockerfile index 26ea2a0..b099b7f 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -3,20 +3,22 @@ LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 testing image" RUN apt-get update && \ - apt-get install -y \ - git wget make && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y \ + git wget make && \ + rm -rf /var/lib/apt/lists/* -RUN wget -q https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz && \ - tar -xf go1.11.5.linux-amd64.tar.gz && \ - rm go1.11.5.linux-amd64.tar.gz && \ - mv go /usr/local +RUN wget -q https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz && \ + tar -xf go1.12.5.linux-amd64.tar.gz && \ + rm go1.12.5.linux-amd64.tar.gz && \ + mv go /usr/local ENV GOROOT /usr/local/go ENV GOPATH /go ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH -RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd +RUN wget -q https://dl.min.io/server/minio/release/linux-amd64/minio && \ + chmod +x minio &&\ + mv minio /usr/local/bin WORKDIR /app diff --git a/test/test.sh b/test/test.sh index 0e8b809..f17838a 100755 --- a/test/test.sh +++ b/test/test.sh @@ -3,6 +3,6 @@ export MINIO_ACCESS_KEY=FJDSJ export MINIO_SECRET_KEY=DSG643HGDS mkdir -p /tmp/minio -minio server --quiet /tmp/minio &>/dev/null & +minio server /tmp/minio &>/dev/null & sleep 5 -go test github.com/ctrox/csi-s3/pkg/s3 -cover -coverprofile=coverage.out \ No newline at end of file +go test github.com/ctrox/csi-s3/pkg/s3 -cover \ No newline at end of file From 152eeb1d1401945fc5e57298ba2c4185572431bf Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Tue, 14 May 2019 21:53:44 +0200 Subject: [PATCH 3/7] Update README and deployment, split up dockerfile --- README.md | 108 +++++++++++++++++++--------- cmd/s3driver/Dockerfile | 65 +++-------------- cmd/s3driver/Dockerfile.full | 68 ++++++++++++++++++ deploy/kubernetes/attacher.yaml | 2 +- deploy/kubernetes/csi-s3.yaml | 9 +-- deploy/kubernetes/pod.yaml | 1 + deploy/kubernetes/provisioner.yaml | 9 +-- deploy/kubernetes/secret.yaml | 6 +- deploy/kubernetes/storageclass.yaml | 10 ++- pkg/s3/s3-driver.go | 2 +- 10 files changed, 168 insertions(+), 112 deletions(-) create mode 100644 cmd/s3driver/Dockerfile.full diff --git a/README.md b/README.md index 1783ac9..4d57413 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,23 @@ # CSI for S3 + This is a Container Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) for S3 (or S3 compatible) storage. This can dynamically allocate buckets and mount them via a fuse mount into any container. -# Status +## Status + This is still very experimental and should not be used in any production environment. Unexpected data loss could occur depending on what mounter and S3 storage backend is being used. -# Kubernetes installation -## Requirements -* Kubernetes 1.10+ +## Kubernetes installation + +### Requirements + +* Kubernetes 1.13+ * Kubernetes has to allow privileged containers * Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`) -## 1. Create a secret with your S3 credentials +### 1. Create a secret with your S3 credentials + The endpoint is optional if you are using something else than AWS S3. Also the region can be empty if you are using some other S3 compatible storage. + ```yaml apiVersion: v1 kind: Secret @@ -20,7 +26,7 @@ metadata: stringData: accessKeyID: secretAccessKey: - # For AWS set it to "https://s3.amazonaws.com" + # For AWS set it to "https://s3..amazonaws.com" endpoint: # If not on S3, set it to "" region: @@ -29,47 +35,60 @@ stringData: encryptionKey: ``` -## 2. Deploy the driver +### 2. Deploy the driver + ```bash cd deploy/kubernetes -$ kubectl create -f provisioner.yaml -$ kubectl create -f attacher.yaml -$ kubectl create -f csi-s3.yaml +kubectl create -f provisioner.yaml +kubectl create -f attacher.yaml +kubectl create -f csi-s3.yaml ``` -## 3. Create the storage class +### 3. Create the storage class + ```bash -$ kubectl create -f storageclass.yaml +kubectl create -f storageclass.yaml ``` -## 4. Test the S3 driver +### 4. Test the S3 driver + * Create a pvc using the new storage class: + ```bash -$ kubectl create -f pvc.yaml +kubectl create -f pvc.yaml ``` + * Check if the PVC has been bound: + ```bash $ kubectl get pvc csi-s3-pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWX csi-s3 9s ``` + * Create a test pod which mounts your volume: + ```bash -$ kubectl create -f poc.yaml +kubectl create -f poc.yaml ``` + If the pod can start, everything should be working. * Test the mount + ```bash $ kubectl exec -ti csi-s3-test-nginx bash $ mount | grep fuse s3fs on /var/lib/www/html type fuse.s3fs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other) $ touch /var/lib/www/html/hello_world ``` + If something does not work as expected, check the troubleshooting section below. -# Additional configuration -## Mounter +## Additional configuration + +### Mounter + As S3 is not a real file system there are some limitations to consider here. Depending on what mounter you are using, you will have different levels of POSIX compability. Also depending on what S3 storage backend you are using there are not always [consistency guarantees](https://github.com/gaul/are-we-consistent-yet#observed-consistency). The driver can be configured to use one of these mounters to mount buckets: @@ -84,22 +103,26 @@ The mounter can be set as a parameter in the storage class. You can also create All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter: -### rclone +#### rclone + * Almost full POSIX compatibility (depends on caching mode) * Files can be viewed normally with any S3 client -### s3fs +#### s3fs + * Large subset of POSIX * Files can be viewed normally with any S3 client * Does not support appends or random writes -### goofys +#### goofys + * Weak POSIX compatibility * Performance first * Files can be viewed normally with any S3 client * Does not support appends or random writes -### s3ql +#### s3ql (not recommended*) + * (Almost) full POSIX compatibility * Uses its own object format * Files are not readable with other S3 clients @@ -107,7 +130,8 @@ All mounters have different strengths and weaknesses depending on your use case. * Supports compression before upload * Supports encryption before upload -### s3backer +#### s3backer (not recommended*) + * Represents a block device stored on S3 * Allows to use a real filesystem * Files are not readable with other S3 clients @@ -115,33 +139,47 @@ All mounters have different strengths and weaknesses depending on your use case. * Supports compression before upload (Not yet implemented in this driver) * Supports encryption before upload (Not yet implemented in this driver) +*s3ql and s3backer are not recommended at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod. + Fore more detailed limitations consult the documentation of the different projects. -# Troubleshooting -## Issues while creating PVC +## Troubleshooting + +### Issues while creating PVC + * Check the logs of the provisioner: + +```bash +kubectl logs -l app=csi-provisioner-s3 -c csi-s3 ``` -$ kubectl logs -l app=csi-provisioner-s3 -c csi-s3 -``` -## Issues creating containers +### Issues creating containers + * Ensure feature gate `MountPropagation` is not set to `false` * Check the logs of the s3-driver: -``` -$ kubectl logs -l app=csi-s3 -c csi-s3 + +```bash +kubectl logs -l app=csi-s3 -c csi-s3 ``` -# Development +## Development + This project can be built like any other go application. + ```bash -$ go get -u github.com/ctrox/csi-s3 +go get -u github.com/ctrox/csi-s3 ``` -## Build + +### Build executable + ```bash -$ make build +make build ``` -## Tests + +### Tests + Currently the driver is tested by the [CSI Sanity Tester](https://github.com/kubernetes-csi/csi-test/tree/master/pkg/sanity). As end-to-end tests require S3 storage and a mounter like s3fs, this is best done in a docker container. A Dockerfile and the test script are in the `test` directory. The easiest way to run the tests is to just use the make command: + ```bash -$ make test +make test ``` diff --git a/cmd/s3driver/Dockerfile b/cmd/s3driver/Dockerfile index f6c94f9..3579a68 100644 --- a/cmd/s3driver/Dockerfile +++ b/cmd/s3driver/Dockerfile @@ -1,68 +1,19 @@ -FROM python:3.6 as s3ql-deps +FROM debian:stretch LABEL maintainers="Cyrill Troxler " -LABEL description="s3ql dependencies" +LABEL description="csi-s3 slim image" RUN apt-get update && \ - apt-get install -y \ - python3 python3-setuptools \ - python3-dev python3-pip pkg-config cython \ - libfuse-dev libattr1-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto - -FROM debian:stretch as s3backer -ARG S3BACKER_VERSION=1.5.0 - -RUN apt-get update && apt-get install -y \ - build-essential \ - autoconf \ - libcurl4-openssl-dev \ - libfuse-dev \ - libexpat1-dev \ - libssl-dev \ - zlib1g-dev \ - psmisc \ - pkg-config \ - git && \ + apt-get install -y \ + s3fs wget unzip && \ rm -rf /var/lib/apt/lists/* -# Compile & install s3backer -RUN git clone https://github.com/archiecobbs/s3backer.git /src/s3backer -WORKDIR /src/s3backer -RUN git checkout tags/${S3BACKER_VERSION} - -RUN ./autogen.sh && \ - ./configure && \ - make && \ - make install - -FROM python:3.6-slim -LABEL maintainers="Cyrill Troxler " -LABEL description="csi-s3 production image" - -RUN apt-get update && \ - apt-get install -y \ - libfuse2 gcc sqlite3 libsqlite3-dev \ - s3fs psmisc procps libcurl3 xfsprogs wget unzip && \ - rm -rf /var/lib/apt/lists/* - -ARG S3QL_VERSION=2.29 -ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2 - -COPY --from=s3ql-deps /root/.cache /root/.cache -COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages -RUN pip install ${S3QL_URL} && rm -rf /root/.cache - -COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer - # install rclone ARG RCLONE_VERSION=v1.46 RUN cd /tmp \ - && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ - && rm -r /tmp/rclone* + && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ + && rm -r /tmp/rclone* COPY ./_output/s3driver /s3driver ENTRYPOINT ["/s3driver"] diff --git a/cmd/s3driver/Dockerfile.full b/cmd/s3driver/Dockerfile.full new file mode 100644 index 0000000..ef266c5 --- /dev/null +++ b/cmd/s3driver/Dockerfile.full @@ -0,0 +1,68 @@ +FROM python:3.6 as s3ql-deps +LABEL maintainers="Cyrill Troxler " +LABEL description="s3ql dependencies" + +RUN apt-get update && \ + apt-get install -y \ + python3 python3-setuptools \ + python3-dev python3-pip pkg-config cython \ + libfuse-dev libattr1-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto + +FROM debian:stretch as s3backer +ARG S3BACKER_VERSION=1.5.0 + +RUN apt-get update && apt-get install -y \ + build-essential \ + autoconf \ + libcurl4-openssl-dev \ + libfuse-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev \ + psmisc \ + pkg-config \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Compile & install s3backer +RUN git clone https://github.com/archiecobbs/s3backer.git /src/s3backer +WORKDIR /src/s3backer +RUN git checkout tags/${S3BACKER_VERSION} + +RUN ./autogen.sh && \ + ./configure && \ + make && \ + make install + +FROM python:3.6-slim +LABEL maintainers="Cyrill Troxler " +LABEL description="csi-s3 full image" + +RUN apt-get update && \ + apt-get install -y \ + libfuse2 gcc sqlite3 libsqlite3-dev \ + s3fs psmisc procps libcurl3 xfsprogs wget unzip && \ + rm -rf /var/lib/apt/lists/* + +ARG S3QL_VERSION=2.29 +ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2 + +COPY --from=s3ql-deps /root/.cache /root/.cache +COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages +RUN pip install ${S3QL_URL} && rm -rf /root/.cache + +COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer + +# install rclone +ARG RCLONE_VERSION=v1.46 +RUN cd /tmp \ + && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ + && rm -r /tmp/rclone* + +COPY ./_output/s3driver /s3driver +ENTRYPOINT ["/s3driver"] diff --git a/deploy/kubernetes/attacher.yaml b/deploy/kubernetes/attacher.yaml index c08998b..5414597 100644 --- a/deploy/kubernetes/attacher.yaml +++ b/deploy/kubernetes/attacher.yaml @@ -67,7 +67,7 @@ spec: serviceAccount: csi-attacher-sa containers: - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v1.0.1 + image: quay.io/k8scsi/csi-attacher:v1.1.0 args: - "--v=4" - "--csi-address=$(ADDRESS)" diff --git a/deploy/kubernetes/csi-s3.yaml b/deploy/kubernetes/csi-s3.yaml index a84a2d8..69bbc5c 100644 --- a/deploy/kubernetes/csi-s3.yaml +++ b/deploy/kubernetes/csi-s3.yaml @@ -55,7 +55,7 @@ spec: hostNetwork: true containers: - name: driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.1 + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 args: - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - "--v=4" @@ -80,15 +80,10 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: ctrox/csi-s3:1.0.1-alpha + image: ctrox/csi-s3:v1.1.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" - - "--access-key-id=$(ACCESS_KEY_ID)" - - "--secret-access-key=$(SECRET_ACCESS_KEY)" - - "--s3-endpoint=$(S3_ENDPOINT)" - - "--region=$(REGION)" - - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT diff --git a/deploy/kubernetes/pod.yaml b/deploy/kubernetes/pod.yaml index 23e4d4b..1e19c95 100644 --- a/deploy/kubernetes/pod.yaml +++ b/deploy/kubernetes/pod.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Pod metadata: name: csi-s3-test-nginx + namespace: default spec: containers: - name: csi-s3-test-nginx diff --git a/deploy/kubernetes/provisioner.yaml b/deploy/kubernetes/provisioner.yaml index 90d5ebe..fd7cc07 100644 --- a/deploy/kubernetes/provisioner.yaml +++ b/deploy/kubernetes/provisioner.yaml @@ -66,7 +66,7 @@ spec: serviceAccount: csi-provisioner-sa containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.0.1 + image: quay.io/k8scsi/csi-provisioner:v1.1.0 args: - "--provisioner=ch.ctrox.csi.s3-driver" - "--csi-address=$(ADDRESS)" @@ -79,15 +79,10 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver - name: csi-s3 - image: ctrox/csi-s3:1.0.1-alpha + image: ctrox/csi-s3:v1.1.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" - - "--access-key-id=$(ACCESS_KEY_ID)" - - "--secret-access-key=$(SECRET_ACCESS_KEY)" - - "--s3-endpoint=$(S3_ENDPOINT)" - - "--region=$(REGION)" - - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 74564d4..033a872 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -5,10 +5,10 @@ metadata: stringData: accessKeyID: secretAccessKey: - # For AWS set it to "https://s3.amazonaws.com" - endpoint: + # For AWS set it to "https://s3..amazonaws.com" + endpoint: https://s3.eu-central-1.amazonaws.com # If not on S3, set it to "" region: # Currently only for s3ql # If not using s3ql, set it to "" - encryptionKey: + encryptionKey: "" diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index 10ace23..eca9192 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -7,4 +7,12 @@ provisioner: ch.ctrox.csi.s3-driver parameters: # specify which mounter to use # can be set to s3backer, s3ql, s3fs or goofys - mounter: s3backer + mounter: rclone + csi.storage.k8s.io/provisioner-secret-name: csi-s3-secret + csi.storage.k8s.io/provisioner-secret-namespace: kube-system + csi.storage.k8s.io/controller-publish-secret-name: csi-s3-secret + csi.storage.k8s.io/controller-publish-secret-namespace: kube-system + csi.storage.k8s.io/node-stage-secret-name: csi-s3-secret + csi.storage.k8s.io/node-stage-secret-namespace: kube-system + csi.storage.k8s.io/node-publish-secret-name: csi-s3-secret + csi.storage.k8s.io/node-publish-secret-namespace: kube-system \ No newline at end of file diff --git a/pkg/s3/s3-driver.go b/pkg/s3/s3-driver.go index 5431829..83f9bbb 100644 --- a/pkg/s3/s3-driver.go +++ b/pkg/s3/s3-driver.go @@ -40,7 +40,7 @@ type s3Volume struct { } var ( - vendorVersion = "1.0.1-alpha" + vendorVersion = "1.1.0" driverName = "ch.ctrox.csi.s3-driver" ) From cc38b816781b37c9b7e1c6d25c45f5aa21b333d9 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Wed, 15 May 2019 19:58:39 +0200 Subject: [PATCH 4/7] Add log message when mounting --- pkg/s3/mounter.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index df17c7a..71fd98e 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -57,6 +57,7 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) { func fuseMount(path string, command string, args []string) error { cmd := exec.Command(command, args...) + glog.V(3).Infof("Mounting fuse with command: %s and args: %s", command, args) out, err := cmd.CombinedOutput() if err != nil { From 76fc704c31d0e98721c19d23e3a84e3cc45286f5 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Wed, 15 May 2019 19:59:18 +0200 Subject: [PATCH 5/7] Do not use sigv2 with s3fs --- pkg/s3/mounter_s3fs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/s3/mounter_s3fs.go b/pkg/s3/mounter_s3fs.go index 772d6c0..082e9fc 100644 --- a/pkg/s3/mounter_s3fs.go +++ b/pkg/s3/mounter_s3fs.go @@ -41,7 +41,6 @@ func (s3fs *s3fsMounter) Mount(source string, target string) error { args := []string{ fmt.Sprintf("%s:/%s", s3fs.bucket.Name, s3fs.bucket.FSPath), fmt.Sprintf("%s", target), - "-o", "sigv2", "-o", "use_path_request_style", "-o", fmt.Sprintf("url=%s", s3fs.url), "-o", fmt.Sprintf("endpoint=%s", s3fs.region), From f80104f215d46b621328113b50bb6b325f34bb91 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Wed, 15 May 2019 20:42:50 +0200 Subject: [PATCH 6/7] Remove s3ql s3ql does not work very well in a dynamic environment like k8s. Also as it needs a ton of dependencies just to get it built makes it hard to maintain. --- Makefile | 7 +- README.md | 26 ++----- cmd/s3driver/Dockerfile | 7 +- cmd/s3driver/Dockerfile.full | 52 ++++--------- deploy/kubernetes/secret.yaml | 3 - deploy/kubernetes/storageclass.yaml | 2 +- pkg/s3/mounter.go | 4 - pkg/s3/mounter_s3ql.go | 114 ---------------------------- pkg/s3/s3-driver_suite_test.go | 27 ------- test/Dockerfile | 2 +- 10 files changed, 34 insertions(+), 210 deletions(-) delete mode 100644 pkg/s3/mounter_s3ql.go diff --git a/Makefile b/Makefile index ab03abf..91a2059 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,9 @@ PROJECT_DIR=/app REGISTRY_NAME=ctrox IMAGE_NAME=csi-s3 -IMAGE_VERSION=1.0.1-alpha -IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) +VERSION ?= dev +IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION) +FULL_IMAGE_TAG=$(IMAGE_TAG)-full TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test build: @@ -27,8 +28,10 @@ test: docker run --rm --privileged -v $(PWD):$(PROJECT_DIR) --device /dev/fuse $(TEST_IMAGE_TAG) container: build docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile . + docker build -t $(FULL_IMAGE_TAG) --build-arg VERSION=$(VERSION) -f cmd/s3driver/Dockerfile.full . push: container docker push $(IMAGE_TAG) + docker push $(FULL_IMAGE_TAG) clean: go clean -r -x -rm -rf _output diff --git a/README.md b/README.md index 4d57413..f36208f 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,12 @@ This is still very experimental and should not be used in any production environ ### Requirements -* Kubernetes 1.13+ +* Kubernetes 1.13+ (CSI v1.0.0 compatibility) * Kubernetes has to allow privileged containers * Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`) ### 1. Create a secret with your S3 credentials -The endpoint is optional if you are using something else than AWS S3. Also the region can be empty if you are using some other S3 compatible storage. - ```yaml apiVersion: v1 kind: Secret @@ -30,11 +28,10 @@ stringData: endpoint: # If not on S3, set it to "" region: - # Currently only for s3ql - # If not using s3ql, set it to "" - encryptionKey: ``` +The region can be empty if you are using some other S3 compatible storage. + ### 2. Deploy the driver ```bash @@ -63,7 +60,7 @@ kubectl create -f pvc.yaml ```bash $ kubectl get pvc csi-s3-pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWX csi-s3 9s +csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWO csi-s3 9s ``` * Create a test pod which mounts your volume: @@ -96,7 +93,6 @@ The driver can be configured to use one of these mounters to mount buckets: * [rclone](https://rclone.org/commands/rclone_mount) * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) * [goofys](https://github.com/kahing/goofys) -* [s3ql](https://github.com/s3ql/s3ql) * [s3backer](https://github.com/archiecobbs/s3backer) The mounter can be set as a parameter in the storage class. You can also create multiple storage classes for each mounter if you like. @@ -121,16 +117,7 @@ All mounters have different strengths and weaknesses depending on your use case. * Files can be viewed normally with any S3 client * Does not support appends or random writes -#### s3ql (not recommended*) - -* (Almost) full POSIX compatibility -* Uses its own object format -* Files are not readable with other S3 clients -* Support appends -* Supports compression before upload -* Supports encryption before upload - -#### s3backer (not recommended*) +#### s3backer (experimental*) * Represents a block device stored on S3 * Allows to use a real filesystem @@ -139,7 +126,8 @@ All mounters have different strengths and weaknesses depending on your use case. * Supports compression before upload (Not yet implemented in this driver) * Supports encryption before upload (Not yet implemented in this driver) -*s3ql and s3backer are not recommended at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod. +*s3backer is experimental at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod. +The s3backer binary is not bundled with the normal docker image to keep that as small as possible. Use the `-full` image tag for testing s3backer. Fore more detailed limitations consult the documentation of the different projects. diff --git a/cmd/s3driver/Dockerfile b/cmd/s3driver/Dockerfile index 3579a68..8e36c1a 100644 --- a/cmd/s3driver/Dockerfile +++ b/cmd/s3driver/Dockerfile @@ -2,15 +2,16 @@ FROM debian:stretch LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 slim image" +# s3fs and some other dependencies RUN apt-get update && \ apt-get install -y \ - s3fs wget unzip && \ + s3fs curl unzip && \ rm -rf /var/lib/apt/lists/* # install rclone -ARG RCLONE_VERSION=v1.46 +ARG RCLONE_VERSION=v1.47.0 RUN cd /tmp \ - && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ && rm -r /tmp/rclone* diff --git a/cmd/s3driver/Dockerfile.full b/cmd/s3driver/Dockerfile.full index ef266c5..58bfcd3 100644 --- a/cmd/s3driver/Dockerfile.full +++ b/cmd/s3driver/Dockerfile.full @@ -1,16 +1,3 @@ -FROM python:3.6 as s3ql-deps -LABEL maintainers="Cyrill Troxler " -LABEL description="s3ql dependencies" - -RUN apt-get update && \ - apt-get install -y \ - python3 python3-setuptools \ - python3-dev python3-pip pkg-config cython \ - libfuse-dev libattr1-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto - FROM debian:stretch as s3backer ARG S3BACKER_VERSION=1.5.0 @@ -33,36 +20,29 @@ WORKDIR /src/s3backer RUN git checkout tags/${S3BACKER_VERSION} RUN ./autogen.sh && \ - ./configure && \ - make && \ - make install + ./configure && \ + make && \ + make install -FROM python:3.6-slim +FROM debian:stretch LABEL maintainers="Cyrill Troxler " -LABEL description="csi-s3 full image" +LABEL description="csi-s3 image" +COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer +# s3fs and some other dependencies RUN apt-get update && \ - apt-get install -y \ - libfuse2 gcc sqlite3 libsqlite3-dev \ - s3fs psmisc procps libcurl3 xfsprogs wget unzip && \ - rm -rf /var/lib/apt/lists/* - -ARG S3QL_VERSION=2.29 -ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2 - -COPY --from=s3ql-deps /root/.cache /root/.cache -COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages -RUN pip install ${S3QL_URL} && rm -rf /root/.cache - -COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer + apt-get install -y \ + libfuse2 gcc sqlite3 libsqlite3-dev \ + s3fs psmisc procps libcurl3 xfsprogs curl unzip && \ + rm -rf /var/lib/apt/lists/* # install rclone -ARG RCLONE_VERSION=v1.46 +ARG RCLONE_VERSION=v1.47.0 RUN cd /tmp \ - && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ - && rm -r /tmp/rclone* + && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ + && rm -r /tmp/rclone* COPY ./_output/s3driver /s3driver ENTRYPOINT ["/s3driver"] diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 033a872..52d03f7 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -9,6 +9,3 @@ stringData: endpoint: https://s3.eu-central-1.amazonaws.com # If not on S3, set it to "" region: - # Currently only for s3ql - # If not using s3ql, set it to "" - encryptionKey: "" diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index eca9192..1bd02ed 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -6,7 +6,7 @@ metadata: provisioner: ch.ctrox.csi.s3-driver parameters: # specify which mounter to use - # can be set to s3backer, s3ql, s3fs or goofys + # can be set to rclone, s3fs, goofys or s3backer mounter: rclone csi.storage.k8s.io/provisioner-secret-name: csi-s3-secret csi.storage.k8s.io/provisioner-secret-namespace: kube-system diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index 71fd98e..0ac2bf9 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -20,7 +20,6 @@ type Mounter interface { const ( s3fsMounterType = "s3fs" goofysMounterType = "goofys" - s3qlMounterType = "s3ql" s3backerMounterType = "s3backer" rcloneMounterType = "rclone" mounterTypeKey = "mounter" @@ -40,9 +39,6 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) { case goofysMounterType: return newGoofysMounter(bucket, cfg) - case s3qlMounterType: - return newS3qlMounter(bucket, cfg) - case s3backerMounterType: return newS3backerMounter(bucket, cfg) diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go deleted file mode 100644 index d32c47d..0000000 --- a/pkg/s3/mounter_s3ql.go +++ /dev/null @@ -1,114 +0,0 @@ -package s3 - -import ( - "bytes" - "fmt" - "net/url" - "os" - "os/exec" - "path" - "strings" - - "gopkg.in/ini.v1" -) - -// Implements Mounter -type s3qlMounter struct { - bucket *bucket - url string - bucketURL string - login string - password string - passphrase string - options []string - ssl bool - targetPath string -} - -const ( - s3qlCmdMkfs = "mkfs.s3ql" - s3qlCmdMount = "mount.s3ql" - s3qlCmdUnmount = "umount.s3ql" -) - -func newS3qlMounter(b *bucket, cfg *Config) (Mounter, error) { - url, err := url.Parse(cfg.Endpoint) - if err != nil { - return nil, err - } - ssl := url.Scheme != "http" - if strings.Contains(url.Scheme, "http") { - url.Scheme = "s3c" - } - s3ql := &s3qlMounter{ - bucket: b, - url: url.String(), - login: cfg.AccessKeyID, - password: cfg.SecretAccessKey, - passphrase: cfg.EncryptionKey, - ssl: ssl, - } - - // s3ql requires a trailing slash or it will just - // prepend the fspath to the s3ql files - url.Path = path.Join(url.Path, b.Name, b.FSPath) + "/" - s3ql.bucketURL = url.String() - - if !ssl { - s3ql.options = []string{"--backend-options", "no-ssl"} - } - - return s3ql, s3ql.writeConfig() -} - -func (s3ql *s3qlMounter) Stage(stagePath string) error { - // force creation to ignore existing data - args := []string{ - s3ql.bucketURL, - "--force", - } - - p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase) - reader := bytes.NewReader([]byte(p)) - cmd := exec.Command(s3qlCmdMkfs, append(args, s3ql.options...)...) - cmd.Stdin = reader - - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("Error running s3ql command: %s", out) - } - return nil -} - -func (s3ql *s3qlMounter) Unstage(stagePath string) error { - return nil -} - -func (s3ql *s3qlMounter) Mount(source string, target string) error { - args := []string{ - s3ql.bucketURL, - target, - "--allow-other", - } - return fuseMount(target, s3qlCmdMount, append(args, s3ql.options...)) -} - -func (s3ql *s3qlMounter) writeConfig() error { - s3qlIni := ini.Empty() - section, err := s3qlIni.NewSection("s3ql") - if err != nil { - return err - } - - section.NewKey("storage-url", s3ql.url) - section.NewKey("backend-login", s3ql.login) - section.NewKey("backend-password", s3ql.password) - section.NewKey("fs-passphrase", s3ql.passphrase) - - authDir := os.Getenv("HOME") + "/.s3ql" - authFile := authDir + "/authinfo2" - os.Mkdir(authDir, 0700) - s3qlIni.SaveTo(authFile) - os.Chmod(authFile, 0600) - return nil -} diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go index dd98fe5..c216e9b 100644 --- a/pkg/s3/s3-driver_suite_test.go +++ b/pkg/s3/s3-driver_suite_test.go @@ -65,33 +65,6 @@ var _ = Describe("S3Driver", func() { }) }) - Context("s3ql", func() { - socket := "/tmp/csi-s3ql.sock" - csiEndpoint := "unix://" + socket - - if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { - Expect(err).NotTo(HaveOccurred()) - } - driver, err := s3.NewS3("test-node", csiEndpoint) - if err != nil { - log.Fatal(err) - } - go driver.Run() - - Describe("CSI sanity", func() { - sanityCfg := &sanity.Config{ - TargetPath: os.TempDir() + "/s3ql-target", - StagingPath: os.TempDir() + "/s3ql-staging", - Address: csiEndpoint, - SecretsFile: "../../test/secret.yaml", - TestVolumeParameters: map[string]string{ - "mounter": "s3ql", - }, - } - sanity.GinkgoTest(sanityCfg) - }) - }) - Context("s3backer", func() { socket := "/tmp/csi-s3backer.sock" csiEndpoint := "unix://" + socket diff --git a/test/Dockerfile b/test/Dockerfile index b099b7f..3cb4237 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,4 +1,4 @@ -FROM ctrox/csi-s3:dev +FROM ctrox/csi-s3:dev-full LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 testing image" From 2d3c8be65bf69daee4a9c24361cc95e2b4115fe8 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Wed, 15 May 2019 21:14:18 +0200 Subject: [PATCH 7/7] Update README --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f36208f..80add03 100644 --- a/README.md +++ b/README.md @@ -49,13 +49,13 @@ kubectl create -f storageclass.yaml ### 4. Test the S3 driver -* Create a pvc using the new storage class: +1. Create a pvc using the new storage class: ```bash kubectl create -f pvc.yaml ``` -* Check if the PVC has been bound: +2. Check if the PVC has been bound: ```bash $ kubectl get pvc csi-s3-pvc @@ -63,7 +63,7 @@ NAME STATUS VOLUME CAPACITY ACC csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWO csi-s3 9s ``` -* Create a test pod which mounts your volume: +3. Create a test pod which mounts your volume: ```bash kubectl create -f poc.yaml @@ -71,7 +71,7 @@ kubectl create -f poc.yaml If the pod can start, everything should be working. -* Test the mount +4. Test the mount ```bash $ kubectl exec -ti csi-s3-test-nginx bash @@ -135,7 +135,7 @@ Fore more detailed limitations consult the documentation of the different projec ### Issues while creating PVC -* Check the logs of the provisioner: +Check the logs of the provisioner: ```bash kubectl logs -l app=csi-provisioner-s3 -c csi-s3 @@ -143,8 +143,8 @@ kubectl logs -l app=csi-provisioner-s3 -c csi-s3 ### Issues creating containers -* Ensure feature gate `MountPropagation` is not set to `false` -* Check the logs of the s3-driver: +1. Ensure feature gate `MountPropagation` is not set to `false` +2. Check the logs of the s3-driver: ```bash kubectl logs -l app=csi-s3 -c csi-s3