Skip to content

Commit

Permalink
Add rclone mounter (#15)
Browse files Browse the repository at this point in the history
  • Loading branch information
ctrox authored Mar 7, 2019
1 parent 9eeb602 commit ea4022e
Show file tree
Hide file tree
Showing 9 changed files with 145 additions and 7 deletions.
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ As S3 is not a real file system there are some limitations to consider here. Dep

The driver can be configured to use one of these mounters to mount buckets:

* [rclone](https://rclone.org/commands/rclone_mount)
* [s3fs](https://github.com/s3fs-fuse/s3fs-fuse)
* [goofys](https://github.com/kahing/goofys)
* [s3ql](https://github.com/s3ql/s3ql)
Expand All @@ -83,6 +84,10 @@ The mounter can be set as a parameter in the storage class. You can also create

All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter:

### rclone
* Almost full POSIX compatibility (depends on caching mode)
* Files can be viewed normally with any S3 client

### s3fs
* Large subset of POSIX
* Files can be viewed normally with any S3 client
Expand Down
10 changes: 9 additions & 1 deletion cmd/s3driver/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ LABEL description="csi-s3 production image"
RUN apt-get update && \
apt-get install -y \
libfuse2 gcc sqlite3 libsqlite3-dev \
s3fs psmisc procps libcurl3 xfsprogs && \
s3fs psmisc procps libcurl3 xfsprogs wget unzip && \
rm -rf /var/lib/apt/lists/*

ARG S3QL_VERSION=2.29
Expand All @@ -56,5 +56,13 @@ RUN pip install ${S3QL_URL} && rm -rf /root/.cache

COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer

# install rclone
ARG RCLONE_VERSION=v1.46
RUN cd /tmp \
&& wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
&& unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
&& mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
&& rm -r /tmp/rclone*

COPY ./_output/s3driver /s3driver
ENTRYPOINT ["/s3driver"]
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
module github.com/ctrox/csi-s3/cmd/s3driver
module github.com/ctrox/csi-s3

require (
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/aws/aws-sdk-go v1.14.27 // indirect
github.com/container-storage-interface/spec v1.0.0
github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096
github.com/go-ini/ini v1.38.1 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
Expand Down
16 changes: 14 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ github.com/aws/aws-sdk-go v1.14.27 h1:fRVME5X3sxZnctdCcabNTWZq7ZGrpVgUAYk4OA5EG0
github.com/aws/aws-sdk-go v1.14.27/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E=
github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096 h1:68EqNg4/ll/Np4mwA4anaM5IlriPqjBaWp3rHJ2wwoM=
github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096/go.mod h1:1CC5YEd7YfNfwmVtjA1okhZTrgYcJCScAllqnYF6fVc=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY=
github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
Expand All @@ -15,21 +14,25 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc=
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb h1:TRAjtOoio6kvnrIMLeXesGT9IydfO+zQoioKWrv40nI=
github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU=
github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbddQ0uScZCHQoZsT5NykGRCeeeIs=
github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kahing/go-xattr v1.1.1 h1:7Ft/P9Gc6iqRVzBRLVw/yLL/dbtzL6FsZzGQj3T9ZY8=
github.com/kahing/go-xattr v1.1.1/go.mod h1:DXZs3JwPmH2DnyFxWjLZWb65lq8pOPtsf9LD+2Gbbpw=
github.com/kahing/goofys v0.19.0 h1:jcuffrnpvZq+LjXtRODo0pvNOglw32ClzBZ1XLShFnk=
github.com/kahing/goofys v0.19.0/go.mod h1:erC9E45nY5m8v6FE+tYIGRVjIC2N8viMlJrgrsXB2Q4=
github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdTUBXB4xE4=
github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0=
github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e h1:BkkRJIF329ps8digiMWthYzDPl9KB8PwkDwvVWDwM4A=
github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrMenWrjM6nZh0Q=
github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
Expand All @@ -41,22 +44,28 @@ github.com/onsi/ginkgo v1.5.0 h1:uZr+v/TFDdYkdA+j02sPO1kA5owrfjBGCJAogfIyThE=
github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.0 h1:p/ZBjQI9G/VwoPrslo/sqS6R5vHU9Od60+axIiP6WuQ=
github.com/onsi/gomega v1.4.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f h1:lv02BiKkf3A85oirJHx0feXbKV4xrq5Nf7QbrNyILoo=
github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA=
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 h1:h7zdf0RiEvWbYBKIx4b+q41xoUVnMmvsGZnIVE5syG8=
golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180712202826-d0887baf81f4 h1:KDF3PK6A+dkI7c4O8QbMtJqcXE3LdNJFGZECIlifQOg=
golang.org/x/net v0.0.0-20180712202826-d0887baf81f4/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180715085529-ac767d655b30 h1:4bYUqrXBoiI7UFQeibUwFhvcHfaEeL75O3lOcZa964o=
golang.org/x/sys v0.0.0-20180715085529-ac767d655b30/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand All @@ -66,8 +75,11 @@ google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b h1:mXqBiicV0B+k8wz
google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.13.0 h1:bHIbVsCwmvbArgCJmLdgOdHFXlKqTOVjbibbS19cXHc=
google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.38.1 h1:8E3nEICVJ6kxl6aTXYp77xYyObhw7YG9/avdj0r3vME=
gopkg.in/ini.v1 v1.38.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
Expand Down
7 changes: 6 additions & 1 deletion pkg/s3/mounter.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package s3
import (
"fmt"
"os/exec"
"time"

"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/mount"
Expand All @@ -22,6 +23,7 @@ const (
goofysMounterType = "goofys"
s3qlMounterType = "s3ql"
s3backerMounterType = "s3backer"
rcloneMounterType = "rclone"
mounterTypeKey = "mounter"
)

Expand All @@ -45,6 +47,9 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) {
case s3backerMounterType:
return newS3backerMounter(bucket, cfg)

case rcloneMounterType:
return newRcloneMounter(bucket, cfg)

default:
// default to s3backer
return newS3backerMounter(bucket, cfg)
Expand All @@ -59,7 +64,7 @@ func fuseMount(path string, command string, args []string) error {
return fmt.Errorf("Error fuseMount command: %s\nargs: %s\noutput: %s", command, args, out)
}

return nil
return waitForMount(path, 10*time.Second)
}

func fuseUnmount(path string, command string) error {
Expand Down
60 changes: 60 additions & 0 deletions pkg/s3/mounter_rclone.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package s3

import (
"fmt"
"os"
)

// Implements Mounter
type rcloneMounter struct {
bucket *bucket
url string
region string
accessKeyID string
secretAccessKey string
}

const (
rcloneCmd = "rclone"
)

func newRcloneMounter(b *bucket, cfg *Config) (Mounter, error) {
return &rcloneMounter{
bucket: b,
url: cfg.Endpoint,
region: cfg.Region,
accessKeyID: cfg.AccessKeyID,
secretAccessKey: cfg.SecretAccessKey,
}, nil
}

func (rclone *rcloneMounter) Stage(stageTarget string) error {
return nil
}

func (rclone *rcloneMounter) Unstage(stageTarget string) error {
return nil
}

func (rclone *rcloneMounter) Mount(source string, target string) error {
args := []string{
"mount",
fmt.Sprintf(":s3:%s/%s", rclone.bucket.Name, rclone.bucket.FSPath),
fmt.Sprintf("%s", target),
"--daemon",
"--s3-provider=AWS",
"--s3-env-auth=true",
fmt.Sprintf("--s3-region=%s", rclone.region),
fmt.Sprintf("--s3-endpoint=%s", rclone.url),
"--allow-other",
// TODO: make this configurable
"--vfs-cache-mode=writes",
}
os.Setenv("AWS_ACCESS_KEY_ID", rclone.accessKeyID)
os.Setenv("AWS_SECRET_ACCESS_KEY", rclone.secretAccessKey)
return fuseMount(target, rcloneCmd, args)
}

func (rclone *rcloneMounter) Unmount(target string) error {
return fuseUnmount(target, rcloneCmd)
}
28 changes: 28 additions & 0 deletions pkg/s3/s3-driver_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,4 +139,32 @@ var _ = Describe("S3Driver", func() {
})
})

Context("rclone", func() {
socket := "/tmp/csi-rclone.sock"
csiEndpoint := "unix://" + socket

cfg := &s3.Config{
AccessKeyID: "FJDSJ",
SecretAccessKey: "DSG643HGDS",
Endpoint: "http://127.0.0.1:9000",
Mounter: "rclone",
}
if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
Expect(err).NotTo(HaveOccurred())
}
driver, err := s3.NewS3("test-node", csiEndpoint, cfg)
if err != nil {
log.Fatal(err)
}
go driver.Run()

Describe("CSI sanity", func() {
sanityCfg := &sanity.Config{
TargetPath: mntDir,
StagingPath: stagingDir,
Address: csiEndpoint,
}
sanity.GinkgoTest(sanityCfg)
})
})
})
21 changes: 21 additions & 0 deletions pkg/s3/util.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package s3

import (
"errors"
"fmt"
"io/ioutil"
"os"
Expand All @@ -10,6 +11,7 @@ import (
"time"

"github.com/mitchellh/go-ps"
"k8s.io/kubernetes/pkg/util/mount"

"github.com/golang/glog"
)
Expand Down Expand Up @@ -39,6 +41,25 @@ func waitForProcess(p *os.Process, backoff int) error {
return waitForProcess(p, backoff+1)
}

func waitForMount(path string, timeout time.Duration) error {
var elapsed time.Duration
var interval = 10 * time.Millisecond
for {
notMount, err := mount.New("").IsNotMountPoint(path)
if err != nil {
return err
}
if !notMount {
return nil
}
time.Sleep(interval)
elapsed = elapsed + interval
if elapsed >= timeout {
return errors.New("Timeout waiting for mount")
}
}
}

func findFuseMountProcess(path string, name string) (*os.Process, error) {
processes, err := ps.Processes()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion test/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM ctrox/csi-s3:1.0.1-alpha
FROM ctrox/csi-s3:dev
LABEL maintainers="Cyrill Troxler <[email protected]>"
LABEL description="csi-s3 testing image"

Expand Down

0 comments on commit ea4022e

Please sign in to comment.