Merge pull request #569 from textileio/sander/buckignore

Adds buckignore and updates threads

buckets/local/bucket.go

@@ -137,14 +137,18 @@ func (b *Bucket) LocalSize() (int64, error) {
 	if err != nil {
 		return 0, err
 	}
+	ig, err := IgnoreFile(bp)
+	if err != nil {
+		return 0, err
+	}
 	var size int64
 	err = filepath.Walk(bp, func(n string, info os.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("getting fileinfo of %s: %s", n, err)
 		}
 		if !info.IsDir() {
 			f := strings.TrimPrefix(n, bp+string(os.PathSeparator))
-			if Ignore(n) || (strings.HasPrefix(f, b.conf.Dir) && f != buckets.SeedName) {
+			if Ignore(n, ig) || (strings.HasPrefix(f, b.conf.Dir) && f != buckets.SeedName) {
 				return nil
 			}
 			size += info.Size()

buckets/local/diff.go

@@ -99,13 +99,21 @@ func (b *Bucket) DiffLocal() ([]Change, error) {
 }
 
 func (b *Bucket) walkPath(pth string) (names []string, err error) {
+	bp, err := b.Path()
+	if err != nil {
+		return
+	}
+	ig, err := IgnoreFile(bp)
+	if err != nil {
+		return
+	}
 	err = filepath.Walk(pth, func(n string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !info.IsDir() {
 			f := strings.TrimPrefix(n, pth+string(os.PathSeparator))
-			if Ignore(n) ||
+			if Ignore(n, ig) ||
 				f == buckets.SeedName ||
 				strings.HasPrefix(f, b.conf.Dir) ||
 				strings.HasSuffix(f, PatchExt) {

buckets/local/repo.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ipfs/go-unixfs/importer/trickle"
 	options "github.com/ipfs/interface-go-ipfs-core/options"
 	mh "github.com/multiformats/go-multihash"
+	ignore "github.com/sabhiram/go-gitignore"
 )
 
 func init() {
@@ -217,12 +218,16 @@ func (b *Repo) recursiveAddPath(
 	if err != nil {
 		return nil, nil, err
 	}
+	ig, err := IgnoreFile(b.path)
+	if err != nil {
+		return nil, nil, err
+	}
 	if err = filepath.Walk(abs, func(n string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 		if !info.IsDir() {
-			if Ignore(n) {
+			if Ignore(n, ig) {
 				return nil
 			}
 			p := n
@@ -435,15 +440,29 @@ func (b *Repo) Close() error {
 }
 
 // Ignore returns true if the path contains an ignored file.
-func Ignore(pth string) bool {
+func Ignore(pth string, ig *ignore.GitIgnore) bool {
 	for _, n := range ignoredFilenames {
 		if strings.HasSuffix(pth, n) {
 			return true
 		}
 	}
+	if ig != nil {
+		return ig.MatchesPath(pth)
+	}
 	return false
 }
 
+// IgnoreFile returns the ignore file at path if it exists
+func IgnoreFile(pth string) (*ignore.GitIgnore, error) {
+	ig, err := ignore.CompileIgnoreFile(filepath.Join(pth, ".buckignore"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	return ig, nil
+}
+
 // addFile chunks reader with layout and adds blocks to the dag service.
 // SHA2-256 is used as the hash function and CidV1 as the cid version.
 func addFile(dag ipld.DAGService, layout options.Layout, prefix cid.Prefix, r io.Reader) (ipld.Node, error) {

cmd/buck/cli/remote.go

@@ -39,6 +39,10 @@ var remoteAddCmd = &cobra.Command{
 		defer cancel()
 		buck, err := bucks.GetLocalBucket(ctx, conf)
 		cmd.ErrCheck(err)
+		bp, err := buck.Path()
+		cmd.ErrCheck(err)
+		ig, err := local.IgnoreFile(bp)
+		cmd.ErrCheck(err)
 
 		var events chan local.Event
 		if !quiet {
@@ -62,7 +66,7 @@ var remoteAddCmd = &cobra.Command{
 			}
 			if !info.IsDir() {
 				f := strings.TrimPrefix(n, pth+string(os.PathSeparator))
-				if local.Ignore(n) ||
+				if local.Ignore(n, ig) ||
 					strings.Contains(f, buckets.SeedName) ||
 					strings.Contains(f, buck.ConfDir()) ||
 					strings.HasSuffix(f, local.PatchExt) {

core/core.go

@@ -268,6 +268,8 @@ func NewTextile(ctx context.Context, conf Config, opts ...Option) (*Textile, err
 	// Configure threads
 	netOptions := []tc.NetOption{
 		tc.WithNetHostAddr(conf.AddrThreadsHost),
+		tc.WithNoNetPulling(true),
+		tc.WithNoExchangeEdgesMigration(true),
 		tc.WithNetDebug(conf.Debug),
 	}
 	if args.ThreadsMongoUri != "" {

go.mod

@@ -62,6 +62,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/cors v1.7.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
@@ -75,7 +76,7 @@ require (
 	github.com/textileio/dcrypto v0.0.1
 	github.com/textileio/go-assets v0.0.0-20200430191519-b341e634e2b7
 	github.com/textileio/go-ds-mongo v0.1.5
-	github.com/textileio/go-threads v1.1.2-0.20211007203214-1842e70155ec
+	github.com/textileio/go-threads v1.1.2-0.20211029155120-3479a196d9b7
 	github.com/textileio/powergate/v2 v2.3.0
 	github.com/textileio/swagger-ui v0.3.29-0.20210224180244-7d73a7a32fe7
 	github.com/xakep666/mongo-migrate v0.2.1

go.sum

@@ -1750,6 +1750,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 h1:ZuhckGJ10ulaKkdvJtiAqsLTiPrLaXSdnVgXJKJkTxE=
@@ -1886,23 +1888,23 @@ github.com/textileio/go-libp2p-pubsub-rpc v0.0.5 h1:De54sqNpQocJebf7P+4RrwtuUw8s
 github.com/textileio/go-libp2p-pubsub-rpc v0.0.5/go.mod h1:MlOMOz3KZxexobvUuFXT/QY9Vjh9eKJpZPr48hDUdVo=
 github.com/textileio/go-log/v2 v2.1.3-gke-1 h1:7e3xSUXQB8hn4uUe5fp41kLThW1o9T65gSM7qjS323g=
 github.com/textileio/go-log/v2 v2.1.3-gke-1/go.mod h1:DwACkjFS3kjZZR/4Spx3aPfSsciyslwUe5bxV8CEU2w=
-github.com/textileio/go-threads v1.1.2-0.20211007203214-1842e70155ec h1:oh/OYfsHLM5cfql71Qi5JYgBk8McSuTCrwGshjqnW00=
-github.com/textileio/go-threads v1.1.2-0.20211007203214-1842e70155ec/go.mod h1:mQzCfbPLpOfHoh07v5u8/i67QRcNjO8DcpS/79LJK6I=
+github.com/textileio/go-threads v1.1.2-0.20211029155120-3479a196d9b7 h1:G984DCftarAL5DjWawFPjS0vi3fhtxRMKNOLEmtUzMM=
+github.com/textileio/go-threads v1.1.2-0.20211029155120-3479a196d9b7/go.mod h1:pC5hdRsNeprQaXVJ9b/EKF/ZCiiYl2KQ6HEAlT/CVDo=
 github.com/textileio/powergate/v2 v2.3.0 h1:kelYh+ZWDQao1rL5YiMznQscd6CsDjgt6P/D1S5UYwQ=
 github.com/textileio/powergate/v2 v2.3.0/go.mod h1:2j2NL1oevaVdrI6MpKfHnfgUOy1D4L7eP3I+1czxDjw=
 github.com/textileio/swagger-ui v0.3.29-0.20210224180244-7d73a7a32fe7 h1:qUEurT6kJF+nFkiNjUPMJJ7hgg9OIDnb8iLn6VtBukE=
 github.com/textileio/swagger-ui v0.3.29-0.20210224180244-7d73a7a32fe7/go.mod h1:IG3de1dcR5Hmcz57nScHHoq5Ju1AABd8z5GnLDgDCks=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
-github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU=
-github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
-github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
-github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
-github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=
-github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.3 h1:5+deguEhHSEjmuICXZ21uSSsXotWMA0orU783+Z7Cp8=
+github.com/tidwall/sjson v1.2.3/go.mod h1:5WdjKx3AQMvCJ4RG6/2UYT7dLrGvJUV1x4jdTAyGvZs=
 github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=