From 321b579a8e2e38074512317152dfa42b5f5f4af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 16 Oct 2024 12:01:25 +0200 Subject: [PATCH] Add flags/config -skip-local-files, -skip-local-dirs Fixes #53 Closes #58 --- README.md | 16 ++-- go.mod | 1 + go.sum | 2 + lib/config.go | 137 +++++++++++++++++++------------ lib/config_test.go | 38 +++++++-- lib/deployer.go | 22 ++--- lib/deployer_test.go | 2 +- lib/files.go | 12 +++ testscripts/skipdirs_custom.txt | 23 ++++++ testscripts/skipdirs_default.txt | 30 +++++++ 10 files changed, 208 insertions(+), 75 deletions(-) create mode 100644 testscripts/skipdirs_custom.txt create mode 100644 testscripts/skipdirs_default.txt diff --git a/README.md b/README.md index 63f8059..fd517b8 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Note that `s3deploy` is a perfect tool to use with a continuous integration tool The list of flags from running `s3deploy -h`: ``` --V print version and exit +-V print version and exit -acl string provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default "private") -bucket string @@ -57,13 +57,13 @@ The list of flags from running `s3deploy -h`: optional config file (default ".s3deploy.yml") -distribution-id value optional CDN distribution ID for cache invalidation, repeat flag for multiple distributions --endpoint-url url - optional AWS endpoint URL override +-endpoint-url string + optional endpoint URL -force upload even if the etags match -h help --ignore string - regexp pattern for ignoring files +-ignore value + regexp pattern for ignoring files, repeat flag for multiple patterns, -key string access key ID for AWS -max-delete int @@ -78,6 +78,10 @@ The list of flags from running `s3deploy -h`: name of AWS region -secret string secret access key for AWS +-skip-local-dirs value + regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default "^\\/?(?:\\w+\\/)*(\\.\\w+)" +-skip-local-files value + regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default "^(.*/)?/?.DS_Store$" -source string path of files to upload (default ".") -try @@ -87,6 +91,8 @@ The list of flags from running `s3deploy -h`: number of workers to upload files (default -1) ``` +Note that `-skip-local-dirs` and `-skip-local-files` will match against a relative path from the source directory with Unix-style path separators. The source directory is represented by `.`, the rest starts with a `/`. + The flags can be set in one of (in priority order): 1. As a flag, e.g. `s3deploy -path public/` diff --git a/go.mod b/go.mod index afb468a..d1e42dd 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 github.com/bep/helpers v0.5.0 + github.com/bep/predicate v0.2.0 github.com/dsnet/golib/memfile v1.0.0 github.com/frankban/quicktest v1.14.6 github.com/oklog/ulid/v2 v2.1.0 diff --git a/go.sum b/go.sum index b654138..b5753a4 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bep/helpers v0.5.0 h1:rneezhnG7GzLFlsEWO/EnleaBRuluBDGFimalO6Y50o= github.com/bep/helpers v0.5.0/go.mod h1:dSqCzIvHbzsk5YOesp1M7sKAq5xUcvANsRoKdawxH4Q= +github.com/bep/predicate v0.2.0 h1:+jHhIbj1UOZn1POqZNKDryuJoi/9wPYg83siaRPb2b0= +github.com/bep/predicate v0.2.0/go.mod h1:MQHXILk/U5Dg7eazQsAB69BrQrYSsl5jLlEejgBQyzg= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/lib/config.go b/lib/config.go index bfb5dc1..2b524b7 100644 --- a/lib/config.go +++ b/lib/config.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/bep/helpers/envhelpers" + "github.com/bep/predicate" "github.com/peterbourgon/ff/v3" "gopkg.in/yaml.v2" ) @@ -44,7 +45,6 @@ func ConfigFromArgs(args []string) (*Config, error) { } return cfg, nil - } // Config configures a deployment. @@ -78,8 +78,17 @@ type Config struct { Silent bool Force bool Try bool - Ignore string - IgnoreRE *regexp.Regexp // compiled version of Ignore + Ignore Strings + + // One or more regular expressions of files to ignore when walking the local directory. + // If not set, defaults to ".DS_Store". + // Note that the path given will have Unix separators, regardless of the OS. + SkipLocalFiles Strings + + // A list of regular expressions of directories to ignore when walking the local directory. + // If not set, defaults to ignoring hidden directories. + // Note that the path given will have Unix separators, regardless of the OS. + SkipLocalDirs Strings // CLI state PrintVersion bool @@ -93,6 +102,11 @@ type Config struct { fs *flag.FlagSet initOnce sync.Once + + // Compiled values. + skipLocalFiles predicate.P[string] + skipLocalDirs predicate.P[string] + ignore predicate.P[string] } func (cfg *Config) Usage() { @@ -108,51 +122,30 @@ func (cfg *Config) Init() error { } func (cfg *Config) loadFileConfig() error { - configFile := cfg.ConfigFile - - if configFile == "" { - return nil - } - - data, err := os.ReadFile(configFile) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - - s := envhelpers.Expand(string(data), func(k string) string { - return os.Getenv(k) - }) - data = []byte(s) - - conf := fileConfig{} - - err = yaml.Unmarshal(data, &conf) - if err != nil { - return err - } - - for _, r := range conf.Routes { - r.routerRE, err = regexp.Compile(r.Route) - + if cfg.ConfigFile != "" { + data, err := os.ReadFile(cfg.ConfigFile) if err != nil { - return err + if !os.IsNotExist(err) { + return err + } + } else { + s := envhelpers.Expand(string(data), func(k string) string { + return os.Getenv(k) + }) + data = []byte(s) + + err = yaml.Unmarshal(data, &cfg.fileConf) + if err != nil { + return err + } } } - cfg.fileConf = conf - - return nil + return cfg.fileConf.init() } func (cfg *Config) shouldIgnoreLocal(key string) bool { - if cfg.Ignore == "" { - return false - } - - return cfg.IgnoreRE.MatchString(key) + return cfg.ignore(key) } func (cfg *Config) shouldIgnoreRemote(key string) bool { @@ -165,13 +158,14 @@ func (cfg *Config) shouldIgnoreRemote(key string) bool { } } - if cfg.Ignore == "" { - return false - } - - return cfg.IgnoreRE.MatchString(sub) + return cfg.ignore(sub) } +const ( + defaultSkipLocalFiles = `^(.*/)?/?.DS_Store$` + defaultSkipLocalDirs = `^\/?(?:\w+\/)*(\.\w+)` +) + func (cfg *Config) init() error { if cfg.BucketName == "" { return errors.New("AWS bucket is required") @@ -209,12 +203,50 @@ func (cfg *Config) init() error { return errors.New("you passed a value for the flags public-access and acl, which is not supported. the public-access flag is deprecated. please use the acl flag moving forward") } - if cfg.Ignore != "" { - re, err := regexp.Compile(cfg.Ignore) + if cfg.Ignore != nil { + for _, pattern := range cfg.Ignore { + re, err := regexp.Compile(pattern) + if err != nil { + return errors.New("cannot compile 'ignore' flag pattern " + err.Error()) + } + fn := func(s string) bool { + return re.MatchString(s) + } + cfg.ignore = cfg.ignore.Or(fn) + } + } else { + cfg.ignore = predicate.P[string](func(s string) bool { + return false + }) + } + + if cfg.SkipLocalFiles == nil { + cfg.SkipLocalFiles = Strings{defaultSkipLocalFiles} + } + if cfg.SkipLocalDirs == nil { + cfg.SkipLocalDirs = Strings{defaultSkipLocalDirs} + } + + for _, pattern := range cfg.SkipLocalFiles { + re, err := regexp.Compile(pattern) if err != nil { - return errors.New("cannot compile 'ignore' flag pattern " + err.Error()) + return err + } + fn := func(s string) bool { + return re.MatchString(s) } - cfg.IgnoreRE = re + cfg.skipLocalFiles = cfg.skipLocalFiles.Or(fn) + } + + for _, pattern := range cfg.SkipLocalDirs { + re, err := regexp.Compile(pattern) + if err != nil { + return err + } + fn := func(s string) bool { + return re.MatchString(s) + } + cfg.skipLocalDirs = cfg.skipLocalDirs.Or(fn) } // load additional config (routes) from file if it exists. @@ -253,7 +285,9 @@ func flagsToConfig(f *flag.FlagSet) *Config { f.BoolVar(&cfg.PublicReadACL, "public-access", false, "DEPRECATED: please set -acl='public-read'") f.StringVar(&cfg.ACL, "acl", "", "provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default \"private\")") f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match") - f.StringVar(&cfg.Ignore, "ignore", "", "regexp pattern for ignoring files") + f.Var(&cfg.Ignore, "ignore", "regexp pattern for ignoring files, repeat flag for multiple patterns,") + f.Var(&cfg.SkipLocalFiles, "skip-local-files", fmt.Sprintf("regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default %q", defaultSkipLocalFiles)) + f.Var(&cfg.SkipLocalDirs, "skip-local-dirs", fmt.Sprintf("regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default %q", defaultSkipLocalDirs)) f.BoolVar(&cfg.Try, "try", false, "trial run, no remote updates") f.BoolVar(&cfg.Verbose, "v", false, "enable verbose logging") f.BoolVar(&cfg.Silent, "quiet", false, "enable silent mode") @@ -343,5 +377,4 @@ func valsToStrs(val interface{}) ([]string, error) { return nil, err } return []string{s}, nil - } diff --git a/lib/config_test.go b/lib/config_test.go index 2730eb0..49ee459 100644 --- a/lib/config_test.go +++ b/lib/config_test.go @@ -51,7 +51,7 @@ func TestConfigFromArgs(t *testing.T) { c.Assert(cfg.Try, qt.Equals, true) c.Assert(cfg.RegionName, qt.Equals, "myregion") c.Assert(cfg.CDNDistributionIDs, qt.DeepEquals, Strings{"mydistro1", "mydistro2"}) - c.Assert(cfg.Ignore, qt.Equals, "^ignored-prefix.*") + c.Assert(cfg.Ignore, qt.DeepEquals, Strings{"^ignored-prefix.*"}) } func TestConfigFromEnvAndFile(t *testing.T) { @@ -66,6 +66,9 @@ func TestConfigFromEnvAndFile(t *testing.T) { bucket: mybucket region: myregion path: ${S3TEST_MYPATH} +ignore: foo +skip-local-dirs: ["a", "b"] +skip-local-files: c routes: - route: "^.+\\.(a)$" @@ -78,7 +81,7 @@ routes: gzip: false - route: "^.+\\.(c)$" gzip: "${S3TEST_GZIP@U}" -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args := []string{ "-config=" + cfgFile, @@ -90,13 +93,15 @@ routes: c.Assert(cfg.BucketName, qt.Equals, "mybucket") c.Assert(cfg.BucketPath, qt.Equals, "mypath") c.Assert(cfg.RegionName, qt.Equals, "myenvregion") + c.Assert(cfg.Ignore, qt.DeepEquals, Strings{"foo"}) + c.Assert(cfg.SkipLocalDirs, qt.DeepEquals, Strings{"a", "b"}) + c.Assert(cfg.SkipLocalFiles, qt.DeepEquals, Strings{"c"}) routes := cfg.fileConf.Routes c.Assert(routes, qt.HasLen, 3) c.Assert(routes[0].Route, qt.Equals, "^.+\\.(a)$") c.Assert(routes[0].Headers["Cache-Control"], qt.Equals, "max-age=1234") c.Assert(routes[0].Gzip, qt.IsTrue) c.Assert(routes[2].Gzip, qt.IsTrue) - } func TestConfigFromFileErrors(t *testing.T) { @@ -105,7 +110,7 @@ func TestConfigFromFileErrors(t *testing.T) { cfgFileInvalidYaml := filepath.Join(dir, "config_invalid_yaml.yml") c.Assert(os.WriteFile(cfgFileInvalidYaml, []byte(` bucket=foo -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args := []string{ "-config=" + cfgFileInvalidYaml, @@ -119,7 +124,7 @@ bucket=foo bucket: foo routes: - route: "*" # invalid regexp. -`), 0644), qt.IsNil) +`), 0o644), qt.IsNil) args = []string{ "-config=" + cfgFileInvalidRoute, @@ -129,7 +134,6 @@ routes: c.Assert(err, qt.IsNil) err = cfg.Init() c.Assert(err, qt.IsNotNil) - } func TestSetAclAndPublicAccessFlag(t *testing.T) { @@ -196,3 +200,25 @@ func TestShouldIgnore(t *testing.T) { c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/any"), qt.IsFalse) c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"), qt.IsTrue) } + +func TestSkipLocalDefault(t *testing.T) { + c := qt.New(t) + + args := []string{ + "-bucket=mybucket", + } + + cfg, err := ConfigFromArgs(args) + c.Assert(err, qt.IsNil) + c.Assert(cfg.Init(), qt.IsNil) + + c.Assert(cfg.skipLocalFiles("foo"), qt.IsFalse) + c.Assert(cfg.skipLocalDirs("foo"), qt.IsFalse) + c.Assert(cfg.skipLocalFiles(".DS_Store"), qt.IsTrue) + c.Assert(cfg.skipLocalFiles("a.DS_Store"), qt.IsFalse) + c.Assert(cfg.skipLocalFiles("foo/bar/.DS_Store"), qt.IsTrue) + + c.Assert(cfg.skipLocalDirs("foo/bar/.git"), qt.IsTrue) + c.Assert(cfg.skipLocalDirs(".git"), qt.IsTrue) + c.Assert(cfg.skipLocalDirs("a.b"), qt.IsFalse) +} diff --git a/lib/deployer.go b/lib/deployer.go index 501d691..af52c15 100644 --- a/lib/deployer.go +++ b/lib/deployer.go @@ -238,34 +238,34 @@ func (d *Deployer) plan(ctx context.Context) error { // walk a local directory func (d *Deployer) walk(ctx context.Context, basePath string, files chan<- *osFile) error { - err := filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(basePath, func(fpath string, info os.FileInfo, err error) error { if err != nil { return err } + pathUnix := path.Clean(filepath.ToSlash(strings.TrimPrefix(fpath, basePath))) + if info.IsDir() { - // skip hidden directories like .git - if path != basePath && strings.HasPrefix(info.Name(), ".") { + if d.cfg.skipLocalDirs(pathUnix) { return filepath.SkipDir } - - return nil - } - - if info.Name() == ".DS_Store" { return nil + } else { + if d.cfg.skipLocalFiles(pathUnix) { + return nil + } } if runtime.GOOS == "darwin" { // When a file system is HFS+, its filepath is in NFD form. - path = norm.NFC.String(path) + fpath = norm.NFC.String(fpath) } - abs, err := filepath.Abs(path) + abs, err := filepath.Abs(fpath) if err != nil { return err } - rel, err := filepath.Rel(basePath, path) + rel, err := filepath.Rel(basePath, fpath) if err != nil { return err } diff --git a/lib/deployer_test.go b/lib/deployer_test.go index 7a7dc07..2761344 100644 --- a/lib/deployer_test.go +++ b/lib/deployer_test.go @@ -116,7 +116,7 @@ func TestDeployWitIgnorePattern(t *testing.T) { Silent: false, SourcePath: source, baseStore: store, - Ignore: re, + Ignore: Strings{re}, } prevCss := m["my/path/main.css"] diff --git a/lib/files.go b/lib/files.go index 4e64b9a..0cb3e9f 100644 --- a/lib/files.go +++ b/lib/files.go @@ -238,6 +238,18 @@ type fileConfig struct { Routes routes `yaml:"routes"` } +func (c *fileConfig) init() error { + for _, r := range c.Routes { + var err error + r.routerRE, err = regexp.Compile(r.Route) + if err != nil { + return err + } + } + + return nil +} + type route struct { Route string `yaml:"route"` Headers map[string]string `yaml:"headers"` diff --git a/testscripts/skipdirs_custom.txt b/testscripts/skipdirs_custom.txt new file mode 100644 index 0000000..7b4aea4 --- /dev/null +++ b/testscripts/skipdirs_custom.txt @@ -0,0 +1,23 @@ +env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY +env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET + +s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/ -skip-local-files 'foo' -skip-local-files bar -skip-local-dirs baz + +stdout 'Deleted 0 of 0, uploaded 2, skipped 0.*100% changed' +stdout 'baz.txt \(not found\) ↑ index.html \(not found\) ↑ $' + +head /$S3DEPLOY_TEST_ID/ +stdout 'Status: 200' + +# By default we skip all . directories and the .DS_Store file. +-- public/index.html -- +Test

Test

+-- public/foo.txt -- +foo content. +-- public/bar.txt -- +bar content. +-- public/baz.txt -- +baz content. +-- public/baz/moo.txt -- +moo content. + diff --git a/testscripts/skipdirs_default.txt b/testscripts/skipdirs_default.txt new file mode 100644 index 0000000..ec12e6f --- /dev/null +++ b/testscripts/skipdirs_default.txt @@ -0,0 +1,30 @@ +env AWS_ACCESS_KEY_ID=$S3DEPLOY_TEST_KEY +env AWS_SECRET_ACCESS_KEY=$S3DEPLOY_TEST_SECRET + +s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=$WORK/public/ + +stdout 'Deleted 0 of 0, uploaded 1, skipped 0.*100% changed' + +head /$S3DEPLOY_TEST_ID/ +stdout 'Status: 200' + + +# Do the same with relative path. +s3deploy -bucket $S3DEPLOY_TEST_BUCKET -region $S3DEPLOY_TEST_REGION -path $S3DEPLOY_TEST_ID -acl 'public-read' -source=public/ + +stdout 'Deleted 0 of 0, uploaded 0, skipped 1 .0% changed' + +head /$S3DEPLOY_TEST_ID/ +stdout 'Status: 200' + + +# By default we skip all . directories and the .DS_Store file. +-- public/index.html -- +Test

Test

+-- public/.hidden/foo.txt -- +foo content. +-- public/.DS_Store -- +binary +-- public/foo/.DS_Store -- +binary +