Fix issue where filepath.Walk does not respect symlinks

.github/workflows/build_and_deploy.yaml

@@ -181,9 +181,9 @@ jobs:
         folder: [
           './examples/container',
          # './examples/build',
-         # './examples/docs',
-          './examples/single_k3s_cluster',
-          './examples/nomad',
+         './examples/docs',
+         # './examples/single_k3s_cluster',
+         # './examples/nomad',
          # './examples/local_exec',
         ]
 

.gitignore

@@ -3,3 +3,4 @@ bin/
 dist/
 .vscode
 .envrc
+.terraform

ChangeLog.md

@@ -1,5 +1,10 @@
 # Change Log
 
+## version v0.5.47
+* Fix isuse where filepath.Walk does not respect symlinks
+* Add `ignore` parameter to `build` resource to allow ignoring of files and folders
+  for Docker builds.
+
 ## version v0.5.43
 * Add ability to set datacenter for Nomad clusters
 * Fix permissions to use octet value when using data_with_permissions

examples/build/build.hcl

@@ -3,11 +3,11 @@ variable "container_enabled" {
 }
 
 variable "nomad_enabled" {
-  default = true
+  default = false
 }
 
 variable "kubernetes_enabled" {
-  default = true
+  default = false
 }
 
 // use a random ingress by default
@@ -27,6 +27,7 @@ resource "build" "app" {
   container {
     dockerfile = "Dockerfile"
     context    = "./src"
+    ignore     = ["**/.terraform"]
   }
 
   output {

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/cucumber/godog v0.12.4
 	github.com/docker/docker v24.0.5+incompatible
 	github.com/docker/go-connections v0.4.0
+	github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222
 	github.com/fatih/color v1.15.0
 	github.com/go-chi/chi v1.5.4
 	github.com/go-chi/cors v1.2.1
@@ -37,14 +38,14 @@ require (
 	github.com/opencontainers/image-spec v1.1.0-rc3
 	github.com/otiai10/copy v1.10.0
 	github.com/pkg/errors v0.9.1
+	github.com/ryanuber/go-glob v1.0.0
 	github.com/sethvargo/go-retry v0.2.4
 	github.com/shipyard-run/gohup v0.2.2
 	github.com/shipyard-run/version-manager v0.0.5
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	github.com/zclconf/go-cty v1.12.0
 	golang.org/x/crypto v0.9.0
-	golang.org/x/mod v0.9.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 	google.golang.org/grpc v1.55.0
 	helm.sh/helm/v3 v3.8.2
@@ -98,6 +99,10 @@ require (
 	github.com/eliukblau/pixterm/pkg/ansimage v0.0.0-20191210081756-9fb6cf8c2f75 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12 // indirect
 	github.com/flytam/filenamify v1.1.1 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect

go.sum

@@ -607,6 +607,16 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
+github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
+github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222 h1:ivxAxcE9py2xLAqpcEwN7sN711aLfEWgh3cY0aha7uY=
+github.com/facebookgo/symwalk v0.0.0-20150726040526-42004b9f3222/go.mod h1:PgrCjL2+FgkITqxQI+erRTONtAv4JkpOzun5ozKW/Jg=
+github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12 h1:pKeuUgeuL6jk/FpxSr0ZVL1XEiOmrcWBvB2rKXu0mMI=
+github.com/facebookgo/testname v0.0.0-20150612200628-5443337c3a12/go.mod h1:IYed2VYeQcs7JTN6KiVXjaz6Rv/Qz092Wjc6o5bCJ9I=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -1332,6 +1342,8 @@ github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNl
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
+github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
 github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
@@ -1608,8 +1620,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
-golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/clients/container/docker_tasks.go

@@ -544,7 +544,7 @@ func (d *DockerTasks) RemoveImage(id string) error {
 
 func (d *DockerTasks) BuildContainer(config *dtypes.Build, force bool) (string, error) {
 	// get the checksum for the id
-	cs, err := utils.HashDir(config.Context)
+	cs, err := utils.HashDir(config.Context, config.Ignore...)
 	if err != nil {
 		return "", err
 	}
@@ -596,7 +596,7 @@ func (d *DockerTasks) BuildContainer(config *dtypes.Build, force bool) (string,
 	}
 
 	var buf bytes.Buffer
-	d.tg.Compress(&buf, &ctar.TarGzOptions{OmitRoot: true}, config.Context)
+	d.tg.Compress(&buf, &ctar.TarGzOptions{OmitRoot: true}, []string{config.Context}, config.Ignore...)
 
 	resp, err := d.c.ImageBuild(context.Background(), &buf, buildOpts)
 	if err != nil {

pkg/clients/container/types/types.go

@@ -84,7 +84,8 @@ type Image struct {
 
 type Build struct {
 	Name       string
-	DockerFile string
-	Context    string
-	Args       map[string]string
+	DockerFile string            // Name of the Dockerfile to use, must be in context
+	Context    string            // Context to copy to the build process
+	Ignore     []string          // globbed list of files to ignore in the context, same as .dockerignore
+	Args       map[string]string // Arguments to pass to the build process
 }

pkg/clients/tar/tar.go

@@ -8,21 +8,28 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/facebookgo/symwalk"
+
+	"github.com/ryanuber/go-glob"
 )
 
 type TarGzOptions struct {
 	// OmitRoot when set to true ignores the top level directory in the tar archive
 	// only adding sub directories and files.
+	// /folder/foo/bar/test.txt -> /test.txt
+	// /folder/foo/bar/baz/* -> /baz
 	OmitRoot bool
 }
 
 type TarGz struct {
 }
 
-// /folder/foo/bar/test.txt -> /test.txt
-// /folder/foo/bar/baz/* -> /baz
+// Compress compresses the src directory into a tar.gz file
+// optionally a list of globs to be ignored can be passed
+func (tg *TarGz) Compress(buf io.Writer, options *TarGzOptions, src []string, ignore ...string) error {
+	var ignoredDirectories []string
 
-func (tg *TarGz) Compress(buf io.Writer, options *TarGzOptions, src ...string) error {
 	if options == nil {
 		options = &TarGzOptions{}
 	}
@@ -45,7 +52,28 @@ func (tg *TarGz) Compress(buf io.Writer, options *TarGzOptions, src ...string) e
 		}
 
 		// walk through every file in the folder
-		filepath.Walk(path, func(file string, fi os.FileInfo, err error) error {
+		// Go's filepath walk does not represent symlinks
+		symwalk.Walk(path, func(file string, fi os.FileInfo, err error) error {
+			// check if the file is in the ignore list
+			for _, i := range ignore {
+				ignore := glob.Glob(i, file)
+				if ignore {
+					// if we are ignoring a directory, we need to also ignore any children
+					// that are in this directory
+					if fi.IsDir() {
+						ignoredDirectories = append(ignoredDirectories, file)
+					}
+
+					return nil
+				}
+			}
+
+			for _, i := range ignoredDirectories {
+				if strings.HasPrefix(file, i) {
+					return nil
+				}
+			}
+
 			// generate tar header
 			header, err := tar.FileInfoHeader(fi, strings.Replace(file, topLevel, "", -1))
 			if err != nil {

pkg/clients/tar/tar_test.go

@@ -53,11 +53,11 @@ func TestTarWithRootFolder(t *testing.T) {
 	tg := &TarGz{}
 
 	// compress the directory
-	err := tg.Compress(buf, nil, in)
+	err := tg.Compress(buf, nil, []string{in})
 	require.NoError(t, err)
 
 	// test the output
-	err = tg.Uncompress(buf, out)
+	err = tg.Uncompress(buf, true, out)
 	require.NoError(t, err)
 
 	require.FileExists(t, filepath.Join(out, "/in/test1.txt"))
@@ -78,11 +78,11 @@ func TestTarOmmitingRoot(t *testing.T) {
 	opts := TarGzOptions{OmitRoot: true}
 
 	// compress the directory
-	err := tg.Compress(buf, &opts, in)
+	err := tg.Compress(buf, &opts, []string{in})
 	require.NoError(t, err)
 
 	// test the output
-	err = tg.Uncompress(buf, out)
+	err = tg.Uncompress(buf, true, out)
 	require.NoError(t, err)
 
 	require.FileExists(t, filepath.Join(out, "/test1.txt"))
@@ -102,11 +102,11 @@ func TestTarIndividualFiles(t *testing.T) {
 	tg := &TarGz{}
 	opts := TarGzOptions{OmitRoot: true}
 
-	err := tg.Compress(buf, &opts, filepath.Join(in, "test1.txt"), filepath.Join(in, "test2.txt"))
+	err := tg.Compress(buf, &opts, []string{filepath.Join(in, "test1.txt"), filepath.Join(in, "test2.txt")})
 	require.NoError(t, err)
 
 	// test the output
-	err = tg.Uncompress(buf, out)
+	err = tg.Uncompress(buf, true, out)
 	require.NoError(t, err)
 
 	require.FileExists(t, filepath.Join(out, "/test1.txt"))
@@ -130,15 +130,38 @@ func TestTarDirAndIndividualFile(t *testing.T) {
 	tg := &TarGz{}
 	opts := TarGzOptions{OmitRoot: true}
 
-	err = tg.Compress(buf, &opts, in, filepath.Join(dir, "solo.txt"))
+	err = tg.Compress(buf, &opts, []string{in, filepath.Join(dir, "solo.txt")})
 	require.NoError(t, err)
 
 	// test the output
-	err = tg.Uncompress(buf, out)
+	err = tg.Uncompress(buf, true, out)
 	require.NoError(t, err)
 
 	require.FileExists(t, filepath.Join(out, "/test1.txt"))
 	require.FileExists(t, filepath.Join(out, "/test2.txt"))
 	require.FileExists(t, filepath.Join(out, "/solo.txt"))
 	require.FileExists(t, filepath.Join(out, "/sub/test3.txt"))
 }
+
+func TestTarDirIgnoringFiles(t *testing.T) {
+	dir := setupTarTests(t)
+
+	in := filepath.Join(dir, "in")
+	out := filepath.Join(dir, "out")
+
+	buf := bytes.NewBuffer(nil)
+
+	tg := &TarGz{}
+	opts := TarGzOptions{OmitRoot: true}
+
+	err := tg.Compress(buf, &opts, []string{in}, "**/test1.txt", "**/sub")
+	require.NoError(t, err)
+
+	// test the output
+	err = tg.Uncompress(buf, true, out)
+	require.NoError(t, err)
+
+	require.NoFileExists(t, filepath.Join(out, "/test1.txt"))
+	require.FileExists(t, filepath.Join(out, "/test2.txt"))
+	require.NoFileExists(t, filepath.Join(out, "/sub/test3.txt"))
+}

pkg/config/resources/build/provider.go

@@ -9,7 +9,6 @@ import (
 	"github.com/jumppad-labs/jumppad/pkg/clients/container/types"
 	"github.com/jumppad-labs/jumppad/pkg/clients/logger"
 	"github.com/jumppad-labs/jumppad/pkg/utils"
-	"golang.org/x/mod/sumdb/dirhash"
 	"golang.org/x/xerrors"
 )
 
@@ -41,7 +40,7 @@ func (b *Provider) Init(cfg htypes.Resource, l logger.Logger) error {
 
 func (b *Provider) Create() error {
 	// calculate the hash
-	hash, err := dirhash.HashDir(b.config.Container.Context, "", dirhash.DefaultHash)
+	hash, err := utils.HashDir(b.config.Container.Context, b.config.Container.Ignore...)
 	if err != nil {
 		return xerrors.Errorf("unable to hash directory: %w", err)
 	}
@@ -64,6 +63,7 @@ func (b *Provider) Create() error {
 		Name:       b.config.Name,
 		DockerFile: b.config.Container.DockerFile,
 		Context:    b.config.Container.Context,
+		Ignore:     b.config.Container.Ignore,
 		Args:       b.config.Container.Args,
 	}
 
@@ -144,7 +144,7 @@ func (b *Provider) Changed() (bool, error) {
 }
 
 func (b *Provider) hasChanged() (bool, error) {
-	hash, err := utils.HashDir(b.config.Container.Context)
+	hash, err := utils.HashDir(b.config.Container.Context, b.config.Container.Ignore...)
 	if err != nil {
 		return false, xerrors.Errorf("unable to hash directory: %w", err)
 	}

pkg/config/resources/build/resource.go

@@ -30,12 +30,13 @@ type Build struct {
 type BuildContainer struct {
 	DockerFile string            `hcl:"dockerfile,optional" json:"dockerfile,omitempty"` // Location of build file inside build context defaults to ./Dockerfile
 	Context    string            `hcl:"context" json:"context"`                          // Path to build context
+	Ignore     []string          `hcl:"ignore,optional" json:"ignore,omitempty"`         // Files to ignore in the build context, this is the same as .dockerignore
 	Args       map[string]string `hcl:"args,optional" json:"args,omitempty"`             // Build args to pass  to the container
 }
 
 type Output struct {
-	Source      string `hcl:"source" json"source"`           // Source file or directory in container
-	Destination string `hcl:"destination" json"destination"` // Destination for copied file or directory
+	Source      string `hcl:"source" json:"source"`           // Source file or directory in container
+	Destination string `hcl:"destination" json:"destination"` // Destination for copied file or directory
 }
 
 func (b *Build) Process() error {