Add flags/config -skip-local-files, -skip-local-dirs

Fixes #53
Closes #58

README.md

@@ -48,7 +48,7 @@ Note that `s3deploy` is a perfect tool to use with a continuous integration tool
 The list of flags from running `s3deploy -h`:
 
 ```
--V print version and exit
+-V	print version and exit
 -acl string
     provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default "private")
 -bucket string
@@ -57,13 +57,13 @@ The list of flags from running `s3deploy -h`:
     optional config file (default ".s3deploy.yml")
 -distribution-id value
     optional CDN distribution ID for cache invalidation, repeat flag for multiple distributions
--endpoint-url url
-	optional AWS endpoint URL override
+-endpoint-url string
+    optional endpoint URL
 -force
     upload even if the etags match
 -h	help
--ignore string
-    regexp pattern for ignoring files
+-ignore value
+    regexp pattern for ignoring files, repeat flag for multiple patterns,
 -key string
     access key ID for AWS
 -max-delete int
@@ -78,6 +78,10 @@ The list of flags from running `s3deploy -h`:
     name of AWS region
 -secret string
     secret access key for AWS
+-skip-local-dirs value
+    regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default "^\\/?(?:\\w+\\/)*(\\.\\w+)"
+-skip-local-files value
+    regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default "^(.*/)?/?.DS_Store$"
 -source string
     path of files to upload (default ".")
 -try
@@ -87,6 +91,8 @@ The list of flags from running `s3deploy -h`:
     number of workers to upload files (default -1)
 ```
 
+Note that `-skip-local-dirs` and `-skip-local-files` will match against a relative path from the source directory with Unix-style path separators. The source directory is represented by `.`, the rest starts with a `/`.
+
 The flags can be set in one of (in priority order):
 
 1. As a flag, e.g. `s3deploy -path public/`

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/cloudfront v1.26.7
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0
 	github.com/bep/helpers v0.5.0
+	github.com/bep/predicate v0.2.0
 	github.com/dsnet/golib/memfile v1.0.0
 	github.com/frankban/quicktest v1.14.6
 	github.com/oklog/ulid/v2 v2.1.0

go.sum

@@ -34,6 +34,8 @@ github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/bep/helpers v0.5.0 h1:rneezhnG7GzLFlsEWO/EnleaBRuluBDGFimalO6Y50o=
 github.com/bep/helpers v0.5.0/go.mod h1:dSqCzIvHbzsk5YOesp1M7sKAq5xUcvANsRoKdawxH4Q=
+github.com/bep/predicate v0.2.0 h1:+jHhIbj1UOZn1POqZNKDryuJoi/9wPYg83siaRPb2b0=
+github.com/bep/predicate v0.2.0/go.mod h1:MQHXILk/U5Dg7eazQsAB69BrQrYSsl5jLlEejgBQyzg=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

lib/config.go

@@ -20,6 +20,7 @@ import (
 	"sync"
 
 	"github.com/bep/helpers/envhelpers"
+	"github.com/bep/predicate"
 	"github.com/peterbourgon/ff/v3"
 	"gopkg.in/yaml.v2"
 )
@@ -44,7 +45,6 @@ func ConfigFromArgs(args []string) (*Config, error) {
 	}
 
 	return cfg, nil
-
 }
 
 // Config configures a deployment.
@@ -78,8 +78,17 @@ type Config struct {
 	Silent          bool
 	Force           bool
 	Try             bool
-	Ignore          string
-	IgnoreRE        *regexp.Regexp // compiled version of Ignore
+	Ignore          Strings
+
+	// One or more regular expressions of files to ignore when walking the local directory.
+	// If not set, defaults to ".DS_Store".
+	// Note that the path given will have Unix separators, regardless of the OS.
+	SkipLocalFiles Strings
+
+	// A list of regular expressions of directories to ignore when walking the local directory.
+	// If not set, defaults to ignoring hidden directories.
+	// Note that the path given will have Unix separators, regardless of the OS.
+	SkipLocalDirs Strings
 
 	// CLI state
 	PrintVersion bool
@@ -93,6 +102,11 @@ type Config struct {
 	fs *flag.FlagSet
 
 	initOnce sync.Once
+
+	// Compiled values.
+	skipLocalFiles predicate.P[string]
+	skipLocalDirs  predicate.P[string]
+	ignore         predicate.P[string]
 }
 
 func (cfg *Config) Usage() {
@@ -108,51 +122,30 @@ func (cfg *Config) Init() error {
 }
 
 func (cfg *Config) loadFileConfig() error {
-	configFile := cfg.ConfigFile
-
-	if configFile == "" {
-		return nil
-	}
-
-	data, err := os.ReadFile(configFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-
-	s := envhelpers.Expand(string(data), func(k string) string {
-		return os.Getenv(k)
-	})
-	data = []byte(s)
-
-	conf := fileConfig{}
-
-	err = yaml.Unmarshal(data, &conf)
-	if err != nil {
-		return err
-	}
-
-	for _, r := range conf.Routes {
-		r.routerRE, err = regexp.Compile(r.Route)
-
+	if cfg.ConfigFile != "" {
+		data, err := os.ReadFile(cfg.ConfigFile)
 		if err != nil {
-			return err
+			if !os.IsNotExist(err) {
+				return err
+			}
+		} else {
+			s := envhelpers.Expand(string(data), func(k string) string {
+				return os.Getenv(k)
+			})
+			data = []byte(s)
+
+			err = yaml.Unmarshal(data, &cfg.fileConf)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
-	cfg.fileConf = conf
-
-	return nil
+	return cfg.fileConf.init()
 }
 
 func (cfg *Config) shouldIgnoreLocal(key string) bool {
-	if cfg.Ignore == "" {
-		return false
-	}
-
-	return cfg.IgnoreRE.MatchString(key)
+	return cfg.ignore(key)
 }
 
 func (cfg *Config) shouldIgnoreRemote(key string) bool {
@@ -165,13 +158,14 @@ func (cfg *Config) shouldIgnoreRemote(key string) bool {
 		}
 	}
 
-	if cfg.Ignore == "" {
-		return false
-	}
-
-	return cfg.IgnoreRE.MatchString(sub)
+	return cfg.ignore(sub)
 }
 
+const (
+	defaultSkipLocalFiles = `^(.*/)?/?.DS_Store$`
+	defaultSkipLocalDirs  = `^\/?(?:\w+\/)*(\.\w+)`
+)
+
 func (cfg *Config) init() error {
 	if cfg.BucketName == "" {
 		return errors.New("AWS bucket is required")
@@ -209,12 +203,50 @@ func (cfg *Config) init() error {
 		return errors.New("you passed a value for the flags public-access and acl, which is not supported. the public-access flag is deprecated. please use the acl flag moving forward")
 	}
 
-	if cfg.Ignore != "" {
-		re, err := regexp.Compile(cfg.Ignore)
+	if cfg.Ignore != nil {
+		for _, pattern := range cfg.Ignore {
+			re, err := regexp.Compile(pattern)
+			if err != nil {
+				return errors.New("cannot compile 'ignore' flag pattern " + err.Error())
+			}
+			fn := func(s string) bool {
+				return re.MatchString(s)
+			}
+			cfg.ignore = cfg.ignore.Or(fn)
+		}
+	} else {
+		cfg.ignore = predicate.P[string](func(s string) bool {
+			return false
+		})
+	}
+
+	if cfg.SkipLocalFiles == nil {
+		cfg.SkipLocalFiles = Strings{defaultSkipLocalFiles}
+	}
+	if cfg.SkipLocalDirs == nil {
+		cfg.SkipLocalDirs = Strings{defaultSkipLocalDirs}
+	}
+
+	for _, pattern := range cfg.SkipLocalFiles {
+		re, err := regexp.Compile(pattern)
 		if err != nil {
-			return errors.New("cannot compile 'ignore' flag pattern " + err.Error())
+			return err
+		}
+		fn := func(s string) bool {
+			return re.MatchString(s)
 		}
-		cfg.IgnoreRE = re
+		cfg.skipLocalFiles = cfg.skipLocalFiles.Or(fn)
+	}
+
+	for _, pattern := range cfg.SkipLocalDirs {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return err
+		}
+		fn := func(s string) bool {
+			return re.MatchString(s)
+		}
+		cfg.skipLocalDirs = cfg.skipLocalDirs.Or(fn)
 	}
 
 	// load additional config (routes) from file if it exists.
@@ -253,7 +285,9 @@ func flagsToConfig(f *flag.FlagSet) *Config {
 	f.BoolVar(&cfg.PublicReadACL, "public-access", false, "DEPRECATED: please set -acl='public-read'")
 	f.StringVar(&cfg.ACL, "acl", "", "provide an ACL for uploaded objects. to make objects public, set to 'public-read'. all possible values are listed here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl (default \"private\")")
 	f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match")
-	f.StringVar(&cfg.Ignore, "ignore", "", "regexp pattern for ignoring files")
+	f.Var(&cfg.Ignore, "ignore", "regexp pattern for ignoring files, repeat flag for multiple patterns,")
+	f.Var(&cfg.SkipLocalFiles, "skip-local-files", fmt.Sprintf("regexp pattern of files to ignore when walking the local directory, repeat flag for multiple patterns, default %q", defaultSkipLocalFiles))
+	f.Var(&cfg.SkipLocalDirs, "skip-local-dirs", fmt.Sprintf("regexp pattern of files of directories to ignore when walking the local directory, repeat flag for multiple patterns, default %q", defaultSkipLocalDirs))
 	f.BoolVar(&cfg.Try, "try", false, "trial run, no remote updates")
 	f.BoolVar(&cfg.Verbose, "v", false, "enable verbose logging")
 	f.BoolVar(&cfg.Silent, "quiet", false, "enable silent mode")
@@ -343,5 +377,4 @@ func valsToStrs(val interface{}) ([]string, error) {
 		return nil, err
 	}
 	return []string{s}, nil
-
 }

lib/config_test.go

@@ -51,7 +51,7 @@ func TestConfigFromArgs(t *testing.T) {
 	c.Assert(cfg.Try, qt.Equals, true)
 	c.Assert(cfg.RegionName, qt.Equals, "myregion")
 	c.Assert(cfg.CDNDistributionIDs, qt.DeepEquals, Strings{"mydistro1", "mydistro2"})
-	c.Assert(cfg.Ignore, qt.Equals, "^ignored-prefix.*")
+	c.Assert(cfg.Ignore, qt.DeepEquals, Strings{"^ignored-prefix.*"})
 }
 
 func TestConfigFromEnvAndFile(t *testing.T) {
@@ -78,7 +78,7 @@ routes:
       gzip: false
     - route: "^.+\\.(c)$"
       gzip: "${S3TEST_GZIP@U}"
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args := []string{
 		"-config=" + cfgFile,
@@ -96,7 +96,6 @@ routes:
 	c.Assert(routes[0].Headers["Cache-Control"], qt.Equals, "max-age=1234")
 	c.Assert(routes[0].Gzip, qt.IsTrue)
 	c.Assert(routes[2].Gzip, qt.IsTrue)
-
 }
 
 func TestConfigFromFileErrors(t *testing.T) {
@@ -105,7 +104,7 @@ func TestConfigFromFileErrors(t *testing.T) {
 	cfgFileInvalidYaml := filepath.Join(dir, "config_invalid_yaml.yml")
 	c.Assert(os.WriteFile(cfgFileInvalidYaml, []byte(`
 bucket=foo
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args := []string{
 		"-config=" + cfgFileInvalidYaml,
@@ -119,7 +118,7 @@ bucket=foo
 bucket: foo
 routes:
    - route: "*" # invalid regexp.
-`), 0644), qt.IsNil)
+`), 0o644), qt.IsNil)
 
 	args = []string{
 		"-config=" + cfgFileInvalidRoute,
@@ -129,7 +128,6 @@ routes:
 	c.Assert(err, qt.IsNil)
 	err = cfg.Init()
 	c.Assert(err, qt.IsNotNil)
-
 }
 
 func TestSetAclAndPublicAccessFlag(t *testing.T) {
@@ -196,3 +194,25 @@ func TestShouldIgnore(t *testing.T) {
 	c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/any"), qt.IsFalse)
 	c.Assert(cfgIgnore.shouldIgnoreRemote("my/path/ignored-prefix/file.txt"), qt.IsTrue)
 }
+
+func TestSkipLocalDefault(t *testing.T) {
+	c := qt.New(t)
+
+	args := []string{
+		"-bucket=mybucket",
+	}
+
+	cfg, err := ConfigFromArgs(args)
+	c.Assert(err, qt.IsNil)
+	c.Assert(cfg.Init(), qt.IsNil)
+
+	c.Assert(cfg.skipLocalFiles("foo"), qt.IsFalse)
+	c.Assert(cfg.skipLocalDirs("foo"), qt.IsFalse)
+	c.Assert(cfg.skipLocalFiles(".DS_Store"), qt.IsTrue)
+	c.Assert(cfg.skipLocalFiles("a.DS_Store"), qt.IsFalse)
+	c.Assert(cfg.skipLocalFiles("foo/bar/.DS_Store"), qt.IsTrue)
+
+	c.Assert(cfg.skipLocalDirs("foo/bar/.git"), qt.IsTrue)
+	c.Assert(cfg.skipLocalDirs(".git"), qt.IsTrue)
+	c.Assert(cfg.skipLocalDirs("a.b"), qt.IsFalse)
+}

lib/deployer.go

@@ -238,34 +238,34 @@ func (d *Deployer) plan(ctx context.Context) error {
 
 // walk a local directory
 func (d *Deployer) walk(ctx context.Context, basePath string, files chan<- *osFile) error {
-	err := filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(basePath, func(fpath string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
+		pathUnix := path.Clean(filepath.ToSlash(strings.TrimPrefix(fpath, basePath)))
+
 		if info.IsDir() {
-			// skip hidden directories like .git
-			if path != basePath && strings.HasPrefix(info.Name(), ".") {
+			if d.cfg.skipLocalDirs(pathUnix) {
 				return filepath.SkipDir
 			}
-
-			return nil
-		}
-
-		if info.Name() == ".DS_Store" {
 			return nil
+		} else {
+			if d.cfg.skipLocalFiles(pathUnix) {
+				return nil
+			}
 		}
 
 		if runtime.GOOS == "darwin" {
 			// When a file system is HFS+, its filepath is in NFD form.
-			path = norm.NFC.String(path)
+			fpath = norm.NFC.String(fpath)
 		}
 
-		abs, err := filepath.Abs(path)
+		abs, err := filepath.Abs(fpath)
 		if err != nil {
 			return err
 		}
-		rel, err := filepath.Rel(basePath, path)
+		rel, err := filepath.Rel(basePath, fpath)
 		if err != nil {
 			return err
 		}