Skip to content

Commit

Permalink
Add support for .slugignore (#18)
Browse files Browse the repository at this point in the history
Closes: #9 

Adds support for `.slugignore` files. `prepare` will now only copy files which do not match any entry in the `.slugignore` file for a given `source-dir`.

The logic is intended to follow logic similar to [this](https://github.com/heroku/slug-compiler/blob/68b63a30907f171e60f8398463d6bbd13b4ed178/lib/slug_compiler.rb#L131-L152):

```ruby
      lines = File.read(slugignore_path).split
      total = lines.inject(0) do |total, line|
        line = (line.split(/#/).first || "").strip
        if line.empty?
          total
        else
          globs = if line =~ /\//
                    [File.join(build_dir, line)]
                  else
                    # 1.8.7 and 1.9.2 handle expanding ** differently,
                    # where in 1.9.2 ** doesn't match the empty case. So
                    # try empty ** explicitly
                    ["", "**"].map { |g| File.join(build_dir, g, line) }
                  end

          to_delete = Dir[*globs].uniq.map { |p| File.expand_path(p) }
          to_delete = to_delete.select { |p| p.match(/^#{build_dir}/) }
          to_delete.each { |p| FileUtils.rm_rf(p) }
          total + to_delete.size
        end
      end
```

Which as far as I can tell has not changes since Heroku closed-source their compiler, which can be verified by extracting the builder source from Heroku via a custom buildpack.
  • Loading branch information
CGA1123 authored Aug 20, 2021
1 parent b8ec89d commit a510838
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 3 deletions.
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ available attached to tagged [releases].
#### `prepare [APPLICATION] --build-dir [BUILD-DIR] --source-dir [SOURCE-DIR]`

In the prepare step, `slugcmplr` will fetch the metadata required to compile
your application. It will copy your project `SOURCE-DIR` into `BUILD-DIR/app`
(it currently do not respect your `.slugcleanup` file, this is TODO).
your application. It will copy your project `SOURCE-DIR` into `BUILD-DIR/app`.

It will fetch the buildpacks as defined by your Heroku application download and
decompress them into `BUILD-DIR/buildpacks`. If using official buildpacks (e.g.
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.16

require (
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
github.com/bmatcuk/doublestar/v4 v4.0.2
github.com/go-git/go-git/v5 v5.4.2
github.com/heroku/heroku-go/v5 v5.3.0
github.com/otiai10/copy v1.6.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down
51 changes: 51 additions & 0 deletions main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"

Expand All @@ -22,6 +23,7 @@ func Test_Suite(t *testing.T) {
// nolint: paralleltest
t.Run("End to end tests", func(t *testing.T) {
t.Run("TestDetectFail", testDetectFail)
t.Run("TestSlugIgnore", testSlugIgnore)
t.Run("TestPrepare", testPrepare)
t.Run("TestGo", testGo)
t.Run("TestRails", testRails)
Expand Down Expand Up @@ -163,6 +165,55 @@ func testDetectFail(t *testing.T) {
})
}

func testSlugIgnore(t *testing.T) {
t.Parallel()

buildpacks := []*BuildpackDescription{
{URL: "https://github.com/CGA1123/heroku-buildpack-bar", Name: "CGA1123/heroku-buildpack-bar"},
{URL: "https://github.com/CGA1123/heroku-buildpack-foo", Name: "CGA1123/heroku-buildpack-foo"},
}

configVars := map[string]string{"FOO": "BAR", "BAR": "FOO"}

withStubPrepare(t, "CGA1123/slugcmplr-fixture-slugignore", buildpacks, configVars, func(t *testing.T, app, buildDir string) {
foundPaths := []string{}
expectedPaths := []string{
"/README.md",
"/.slugignore",
"/keep-me/hello.txt",
"/vendor/keep-this-dir/file-1.txt",
"/vendor/keep-this-dir/file-2.txt",
}

err := filepath.Walk(filepath.Join(buildDir, buildpack.AppDir), func(path string, info os.FileInfo, err error) error {
if err != nil {
t.Fatalf("error while walking directory: %v", err)
}

if info.Mode().IsRegular() {
foundPaths = append(foundPaths, strings.TrimPrefix(path, filepath.Join(buildDir, buildpack.AppDir)))
}

return nil
})
if err != nil {
t.Fatalf("filepath.Walk error: %v", err)
}

sort.Strings(foundPaths)
sort.Strings(expectedPaths)

if !SliceEqual(foundPaths, expectedPaths, func(i int) bool {
return foundPaths[i] == expectedPaths[i]
}) {
expected := strings.Join(expectedPaths, "\n")
actual := strings.Join(foundPaths, "\n")

t.Fatalf("\nexpected:\n%v\n---\nactual:\n%v\n", expected, actual)
}
})
}

func testBinary(t *testing.T) {
t.Parallel()

Expand Down
15 changes: 14 additions & 1 deletion prepare.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import (
"os"
"path/filepath"
"sort"
"strings"

"github.com/cga1123/slugcmplr/buildpack"
"github.com/cga1123/slugcmplr/slugignore"
"github.com/otiai10/copy"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -100,8 +102,19 @@ func prepare(ctx context.Context, cmd Outputter, p *Prepare) error {
log(cmd, "From: %v", p.SourceDir)
log(cmd, "To: %v", appDir)

ignore, err := slugignore.ForDirectory(p.SourceDir)
if err != nil {
return fmt.Errorf("failed to read .slugignore: %v", err)
}

// copy source
if err := copy.Copy(p.SourceDir, appDir); err != nil {
if err := copy.Copy(p.SourceDir, appDir, copy.Options{
Skip: func(path string) (bool, error) {
return ignore.IsIgnored(
strings.TrimPrefix(path, p.SourceDir),
), nil
},
}); err != nil {
return fmt.Errorf("failed to copy source: %w", err)
}

Expand Down
113 changes: 113 additions & 0 deletions slugignore/slugignore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// package slugignore implements Heroku-like .slugignore functionality for a
// given directory.
//
// Heroku's .slugignore format treats all non-empty and non comment lines
// (comment lines are those beginning with a # characted) as Ruby Dir globs,
package slugignore

import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/bmatcuk/doublestar/v4"
)

// SlugIgnore is the interface for a parsed .slugignore file
//
// A given SlugIgnore is only applicable to the directory which contains the
// .slugignore file, at the time at which it was parsed.
type SlugIgnore interface {
IsIgnored(path string) bool
}

// ForDirectory parses the .slugignore for a given directory.
//
// If there is no .slugignore file found, it returns a SlugIgnore which always
// returns false when IsIgnored is called.
func ForDirectory(dir string) (SlugIgnore, error) {
f, err := os.Open(filepath.Join(dir, ".slugignore"))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return &nullSlugIgnore{}, nil
}

return nil, err
}
defer f.Close()

s := bufio.NewScanner(bufio.NewReader(f))
globs := []string{}

for s.Scan() {
line := s.Text()
if strings.HasPrefix(line, "#") {
continue
}

if strings.TrimSpace(line) == "" {
continue
}

trimmed := strings.TrimPrefix(line, "/")
if strings.HasPrefix(line, "/") {
globs = append(globs, trimmed)
} else {
globs = append(
globs,
trimmed,
filepath.Join("**", trimmed),
)
}
}
if err := s.Err(); err != nil {
return nil, fmt.Errorf("error parsing .slugignore: %w", err)
}

ignored := map[string]struct{}{}
for _, glob := range globs {
if !doublestar.ValidatePattern(glob) {
return nil, fmt.Errorf("slugignore pattern is malformed: %v", glob)
}

matches, err := doublestar.Glob(os.DirFS(dir), glob)
if err != nil {
return nil, fmt.Errorf("error expanding glob %v: %w", glob, err)
}

for _, match := range matches {
ignored[match] = struct{}{}
}
}

return cache(ignored), nil
}

type nullSlugIgnore struct{}

func (*nullSlugIgnore) IsIgnored(path string) bool {
return false
}

type cache map[string]struct{}

func (c cache) IsIgnored(path string) bool {
path = strings.TrimPrefix(path, "/")

for {
if _, ok := c[path]; ok {
return true
}

path = filepath.Join(path, "..")

if path == "." {
break
}
}

return false
}

0 comments on commit a510838

Please sign in to comment.