diff --git a/src/build/BUILD b/src/build/BUILD index 28503c0f76..146e9f1486 100644 --- a/src/build/BUILD +++ b/src/build/BUILD @@ -48,6 +48,7 @@ go_test( "///third_party/go/github.com_stretchr_testify//require", "//src/cli/logging", "//src/core", + "//src/fs", "//src/output", "//src/plz", ], diff --git a/src/build/build_step.go b/src/build/build_step.go index 944bd329ca..3c5d808158 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -318,7 +318,7 @@ func buildTarget(state *core.BuildState, target *core.BuildTarget, runRemotely b // Add optional outputs to target metadata metadata.OptionalOutputs = make([]string, 0) - for _, output := range fs.Glob(state.Config.Parse.BuildFileName, target.TmpDir(), target.OptionalOutputs, nil, true) { + for _, output := range fs.Glob(fs.HostFS, state.Config.Parse.BuildFileName, target.TmpDir(), target.OptionalOutputs, nil, true) { log.Debug("Add discovered optional output to metadata %s", output) metadata.OptionalOutputs = append(metadata.OptionalOutputs, output) } @@ -709,7 +709,7 @@ func moveOutputs(state *core.BuildState, target *core.BuildTarget) ([]string, bo } // Optional outputs get moved but don't contribute to the hash or for incrementality. // Glob patterns are supported on these. - for _, output := range fs.Glob(state.Config.Parse.BuildFileName, tmpDir, target.OptionalOutputs, nil, true) { + for _, output := range fs.Glob(fs.HostFS, state.Config.Parse.BuildFileName, tmpDir, target.OptionalOutputs, nil, true) { log.Debug("Discovered optional output %s", output) tmpOutput := filepath.Join(tmpDir, output) realOutput := filepath.Join(outDir, output) diff --git a/src/build/build_step_stress_test.go b/src/build/build_step_stress_test.go index 5b8a290cb3..65d4040cf6 100644 --- a/src/build/build_step_stress_test.go +++ b/src/build/build_step_stress_test.go @@ -12,6 +12,7 @@ import ( "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/plz" ) @@ -22,7 +23,7 @@ const size = 1000 var state *core.BuildState func TestBuildLotsOfTargets(t *testing.T) { - config, _ := core.ReadConfigFiles(core.HostFS(), nil, nil) + config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) config.Please.NumThreads = 10 state = core.NewBuildState(config) state.Parser = &fakeParser{ diff --git a/src/build/build_step_test.go b/src/build/build_step_test.go index fd6736acdb..309fb6fb20 100644 --- a/src/build/build_step_test.go +++ b/src/build/build_step_test.go @@ -519,7 +519,7 @@ func TestSha1SingleHash(t *testing.T) { } func newStateWithHashCheckers(label, hashFunction string, hashCheckers ...string) (*core.BuildState, *core.BuildTarget) { - config, _ := core.ReadConfigFiles(core.HostFS(), nil, nil) + config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) if hashFunction != "" { config.Build.HashFunction = hashFunction } @@ -538,7 +538,7 @@ func newStateWithHashCheckers(label, hashFunction string, hashCheckers ...string } func newStateWithHashFunc(label, hashFunc string) (*core.BuildState, *core.BuildTarget) { - config, _ := core.ReadConfigFiles(core.HostFS(), nil, nil) + config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) config.Build.HashFunction = hashFunc state := core.NewBuildState(config) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} @@ -552,7 +552,7 @@ func newStateWithHashFunc(label, hashFunc string) (*core.BuildState, *core.Build } func newState(label string) (*core.BuildState, *core.BuildTarget) { - config, _ := core.ReadConfigFiles(core.HostFS(), nil, nil) + config, _ := core.ReadConfigFiles(fs.HostFS, nil, nil) state := core.NewBuildState(config) state.Config.Parse.BuildFileName = []string{"BUILD_FILE"} target := core.NewBuildTarget(core.ParseBuildLabel(label, "")) diff --git a/src/core/config_test.go b/src/core/config_test.go index 9f5316e2b4..650e2b6a57 100644 --- a/src/core/config_test.go +++ b/src/core/config_test.go @@ -13,11 +13,12 @@ import ( "github.com/thought-machine/go-flags" "github.com/thought-machine/please/src/cli" + "github.com/thought-machine/please/src/fs" ) func TestPlzConfigWorking(t *testing.T) { RepoRoot = "/repo/root" - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/working.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/working.plzconfig"}, nil) assert.NoError(t, err) assert.Equal(t, "pexmabob", config.Python.PexTool) @@ -31,12 +32,12 @@ func TestPlzConfigWorking(t *testing.T) { } func TestPlzConfigFailing(t *testing.T) { - _, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/failing.plzconfig"}, nil) + _, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/failing.plzconfig"}, nil) assert.Error(t, err) } func TestPlzConfigProfile(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/working.plzconfig"}, []string{"dev"}) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/working.plzconfig"}, []string{"dev"}) assert.NoError(t, err) assert.Equal(t, "pexmabob", config.Python.PexTool) assert.Equal(t, "/opt/java/bin/javac", config.Java.JavacTool) @@ -46,7 +47,7 @@ func TestPlzConfigProfile(t *testing.T) { } func TestMultiplePlzConfigFiles(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{ + config, err := ReadConfigFiles(fs.HostFS, []string{ "src/core/test_data/working.plzconfig", "src/core/test_data/failing.plzconfig", }, nil) @@ -56,7 +57,7 @@ func TestMultiplePlzConfigFiles(t *testing.T) { } func TestConfigSlicesOverwrite(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/slices.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/slices.plzconfig"}, nil) assert.NoError(t, err) // This should be completely overwritten by the config file assert.Equal(t, []string{"/sbin"}, config.Build.Path) @@ -175,29 +176,29 @@ func TestPleaseTildeLocationOverride(t *testing.T) { } func TestReadSemver(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/version_good.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/version_good.plzconfig"}, nil) assert.NoError(t, err) assert.EqualValues(t, 2, config.Please.Version.Major) assert.EqualValues(t, 3, config.Please.Version.Minor) assert.EqualValues(t, 4, config.Please.Version.Patch) - _, err = ReadConfigFiles(HostFS(), []string{"src/core/test_data/version_bad.plzconfig"}, nil) + _, err = ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/version_bad.plzconfig"}, nil) assert.Error(t, err) } func TestReadDurations(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/duration_good.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/duration_good.plzconfig"}, nil) assert.NoError(t, err) assert.EqualValues(t, 500*time.Millisecond, config.Build.Timeout) assert.EqualValues(t, 5*time.Second, config.Test.Timeout) - _, err = ReadConfigFiles(HostFS(), []string{"src/core/test_data/duration_bad.plzconfig"}, nil) + _, err = ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/duration_bad.plzconfig"}, nil) assert.Error(t, err) } func TestReadByteSizes(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/bytesize_good.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/bytesize_good.plzconfig"}, nil) assert.NoError(t, err) assert.EqualValues(t, 500*1000*1000, config.Cache.DirCacheHighWaterMark) - _, err = ReadConfigFiles(HostFS(), []string{"src/core/test_data/bytesize_bad.plzconfig"}, nil) + _, err = ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/bytesize_bad.plzconfig"}, nil) assert.Error(t, err) } @@ -210,29 +211,29 @@ func TestCompletions(t *testing.T) { } func TestConfigVerifiesOptions(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/testrunner_good.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/testrunner_good.plzconfig"}, nil) assert.NoError(t, err) assert.Equal(t, "pytest", config.Python.TestRunner) - _, err = ReadConfigFiles(HostFS(), []string{"src/core/test_data/testrunner_bad.plzconfig"}, nil) + _, err = ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/testrunner_bad.plzconfig"}, nil) assert.Error(t, err) } func TestDefaultHashCheckers(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), nil, nil) + config, err := ReadConfigFiles(fs.HostFS, nil, nil) assert.NoError(t, err) assert.ElementsMatch(t, []string{"sha1", "sha256", "blake3"}, config.Build.HashCheckers) } func TestHashCheckersConfig(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/hashcheckers.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/hashcheckers.plzconfig"}, nil) assert.NoError(t, err) assert.ElementsMatch(t, []string{"blake3"}, config.Build.HashCheckers) } func TestOverrideHashCheckersConfig(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/hashcheckers.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/hashcheckers.plzconfig"}, nil) assert.NoError(t, err) err = config.ApplyOverrides(map[string]string{"build.hashcheckers": "sha256"}) @@ -241,7 +242,7 @@ func TestOverrideHashCheckersConfig(t *testing.T) { } func TestOverrideHashCheckersNoConfig(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), nil, nil) + config, err := ReadConfigFiles(fs.HostFS, nil, nil) assert.NoError(t, err) err = config.ApplyOverrides(map[string]string{"build.hashcheckers": "sha1,blake3"}) @@ -250,7 +251,7 @@ func TestOverrideHashCheckersNoConfig(t *testing.T) { } func TestUnknownHashChecker(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), nil, nil) + config, err := ReadConfigFiles(fs.HostFS, nil, nil) assert.NoError(t, err) err = config.ApplyOverrides(map[string]string{"build.hashcheckers": "fake-algo"}) @@ -258,7 +259,7 @@ func TestUnknownHashChecker(t *testing.T) { } func TestBuildEnvSection(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/buildenv.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/buildenv.plzconfig"}, nil) assert.NoError(t, err) expected := []string{ "BAR_BAR=first", @@ -271,7 +272,7 @@ func TestBuildEnvSection(t *testing.T) { func TestPassEnv(t *testing.T) { t.Setenv("FOO", "first") t.Setenv("BAR", "second") - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/passenv.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/passenv.plzconfig"}, nil) assert.NoError(t, err) expected := []string{ "BAR=second", @@ -284,7 +285,7 @@ func TestPassEnv(t *testing.T) { func TestPassUnsafeEnv(t *testing.T) { t.Setenv("FOO", "first") t.Setenv("BAR", "second") - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/passunsafeenv.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/passunsafeenv.plzconfig"}, nil) assert.NoError(t, err) expected := []string{ "BAR=second", @@ -301,7 +302,7 @@ func TestPassUnsafeEnvExcludedFromHash(t *testing.T) { err = os.Unsetenv("BAR") require.NoError(t, err) - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/passunsafeenv.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/passunsafeenv.plzconfig"}, nil) require.NoError(t, err) expected := config.Hash() @@ -313,7 +314,7 @@ func TestPassUnsafeEnvExcludedFromHash(t *testing.T) { } func TestBuildPathWithPathEnv(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/passenv.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/passenv.plzconfig"}, nil) assert.NoError(t, err) assert.Equal(t, config.Build.Path, strings.Split(os.Getenv("PATH"), ":")) } @@ -348,7 +349,7 @@ func TestUpdateArgsWithQuotedAliases(t *testing.T) { } func TestParseNewFormatAliases(t *testing.T) { - c, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/alias.plzconfig"}, nil) + c, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/alias.plzconfig"}, nil) assert.NoError(t, err) assert.Equal(t, 2, len(c.Alias)) a := c.Alias["auth"] @@ -358,7 +359,7 @@ func TestParseNewFormatAliases(t *testing.T) { } func TestAttachAliasFlags(t *testing.T) { - c, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/alias.plzconfig"}, nil) + c, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/alias.plzconfig"}, nil) assert.NoError(t, err) t.Setenv("GO_FLAGS_COMPLETION", "1") p := flags.NewParser(&struct{}{}, 0) @@ -394,7 +395,7 @@ func TestAttachAliasFlags(t *testing.T) { } func TestPrintAliases(t *testing.T) { - c, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/alias.plzconfig"}, nil) + c, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/alias.plzconfig"}, nil) assert.NoError(t, err) var buf bytes.Buffer c.PrintAliases(&buf) @@ -436,7 +437,7 @@ func TestEnsurePleaseLocation(t *testing.T) { } func TestPluginConfig(t *testing.T) { - config, err := ReadConfigFiles(HostFS(), []string{"src/core/test_data/plugin.plzconfig"}, nil) + config, err := ReadConfigFiles(fs.HostFS, []string{"src/core/test_data/plugin.plzconfig"}, nil) assert.NoError(t, err) assert.Equal(t, []string{"fooc"}, config.Plugin["foo"].ExtraValues["fooctool"]) } diff --git a/src/core/state.go b/src/core/state.go index db7827dde4..71367d139a 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -1192,12 +1192,12 @@ func (state *BuildState) ForArch(arch cli.Arch) *BuildState { configPath := ".plzconfig_" + arch.String() config := state.Config.copyConfig() - if err := readConfigFile(HostFS(), config, configPath, false); err != nil { + if err := readConfigFile(fs.HostFS, config, configPath, false); err != nil { log.Fatalf("%v", err) } repoConfig := state.Config.copyConfig() - if err := readConfigFile(HostFS(), repoConfig, configPath, false); err != nil { + if err := readConfigFile(fs.HostFS, repoConfig, configPath, false); err != nil { log.Fatalf("%v", err) } diff --git a/src/core/subrepo.go b/src/core/subrepo.go index fcb4e32327..eb52609215 100644 --- a/src/core/subrepo.go +++ b/src/core/subrepo.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/thought-machine/please/src/cli" + "github.com/thought-machine/please/src/fs" ) // A Subrepo stores information about a registered subrepository, typically one @@ -82,7 +83,7 @@ func (s *Subrepo) Dir(dir string) string { func readConfigFilesInto(repoConfig *Configuration, files []string) error { for _, file := range files { - err := readConfigFile(HostFS(), repoConfig, file, true) + err := readConfigFile(fs.HostFS, repoConfig, file, true) if err != nil { return err } diff --git a/src/core/utils.go b/src/core/utils.go index cd797b202d..a6d7b97169 100644 --- a/src/core/utils.go +++ b/src/core/utils.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/sha1" "fmt" - iofs "io/fs" "os" "path/filepath" "strings" @@ -37,17 +36,6 @@ func FindRepoRoot() bool { return RepoRoot != "" } -type osFS func(name string) (*os.File, error) - -func (r osFS) Open(name string) (iofs.File, error) { - return r(name) -} - -// HostFS returns an io/fs.FS that behaves the same as the host OS i.e. the same way os.Open works. -func HostFS() iofs.FS { - return osFS(os.Open) -} - // MustFindRepoRoot returns the root directory of the current repo and sets the initial working dir. // It dies on failure, although will fall back to looking for a Bazel WORKSPACE file first. func MustFindRepoRoot() string { @@ -65,7 +53,7 @@ func MustFindRepoRoot() string { } // Check the config for a default repo location. Of course, we have to load system-level config // in order to do that... - config, err := ReadConfigFiles(HostFS(), defaultGlobalConfigFiles(), nil) + config, err := ReadConfigFiles(fs.HostFS, defaultGlobalConfigFiles(), nil) if err != nil { log.Fatalf("Error reading config file: %s", err) } diff --git a/src/fs/BUILD b/src/fs/BUILD index 85c61bda72..1f6f35a792 100644 --- a/src/fs/BUILD +++ b/src/fs/BUILD @@ -10,6 +10,7 @@ go_library( "home.go", "sort.go", "walk.go", + "iofs.go", ], pgo_file = "//:pgo", visibility = ["PUBLIC"], diff --git a/src/fs/glob.go b/src/fs/glob.go index 59b824f59c..4c98ee345a 100644 --- a/src/fs/glob.go +++ b/src/fs/glob.go @@ -2,6 +2,7 @@ package fs import ( "fmt" + iofs "io/fs" "path/filepath" "regexp" "strings" @@ -63,14 +64,15 @@ func IsGlob(pattern string) bool { // Glob implements matching using Go's built-in filepath.Glob, but extends it to support // Ant-style patterns using **. -func Glob(buildFileNames []string, rootPath string, includes, excludes []string, includeHidden bool) []string { - return NewGlobber(buildFileNames).Glob(rootPath, includes, excludes, includeHidden, true) +func Glob(fs iofs.FS, buildFileNames []string, rootPath string, includes, excludes []string, includeHidden bool) []string { + return NewGlobber(fs, buildFileNames).Glob(rootPath, includes, excludes, includeHidden, true) } // A Globber is used to implement Glob. You can persist one for use to save repeated filesystem calls, but // it isn't safe for use in concurrent goroutines. type Globber struct { buildFileNames []string + fs iofs.ReadDirFS walkedDirs map[string]walkedDir } @@ -87,9 +89,14 @@ func Match(glob, path string) (bool, error) { } // NewGlobber creates a new Globber. You should call this rather than creating one directly (or use Glob() if you don't care). -func NewGlobber(buildFileNames []string) *Globber { +func NewGlobber(fs iofs.FS, buildFileNames []string) *Globber { + rdfs, ok := fs.(iofs.ReadDirFS) + if !ok { + log.Fatalf("NewGlobber must be constructed with a ReadDirFS") + } return &Globber{ buildFileNames: buildFileNames, + fs: rdfs, walkedDirs: map[string]walkedDir{}, } } @@ -165,25 +172,27 @@ func (globber *Globber) walkDir(rootPath string) (walkedDir, error) { return dir, nil } dir := walkedDir{} - if err := WalkMode(rootPath, func(name string, mode Mode) error { - if isBuildFile(globber.buildFileNames, name) { - packageName := filepath.Dir(name) + err := iofs.WalkDir(globber.fs, rootPath, func(path string, d iofs.DirEntry, err error) error { + typeMode := mode(d.Type()) + if isBuildFile(globber.buildFileNames, path) { + packageName := filepath.Dir(path) if packageName != rootPath { dir.subPackages = append(dir.subPackages, packageName) return filepath.SkipDir } } // Exclude plz-out - if name == "plz-out" && rootPath == "." { + if d.Name() == "plz-out" && rootPath == "." { return filepath.SkipDir } - if mode.IsSymlink() { - dir.symlinks = append(dir.symlinks, name) + if typeMode.IsSymlink() { + dir.symlinks = append(dir.symlinks, path) } else { - dir.fileNames = append(dir.fileNames, name) + dir.fileNames = append(dir.fileNames, path) } return nil - }); err != nil { + }) + if err != nil { return dir, err } globber.walkedDirs[rootPath] = dir diff --git a/src/fs/glob_test.go b/src/fs/glob_test.go index 60a86641b0..e9a8ee7189 100644 --- a/src/fs/glob_test.go +++ b/src/fs/glob_test.go @@ -13,7 +13,7 @@ import ( var buildFileNames = []string{"TEST_BUILD", "BUILD"} func glob(rootPath string, glob string, excludes []string, includeHidden bool) ([]string, error) { - return NewGlobber(buildFileNames).glob(rootPath, glob, excludes, includeHidden, true) + return NewGlobber(HostFS, buildFileNames).glob(rootPath, glob, excludes, includeHidden, true) } func TestCanGlobFileAtRootWithDoubleStar(t *testing.T) { @@ -72,7 +72,7 @@ func TestGlobPlusPlusInDirName(t *testing.T) { func TestGlobExcludes(t *testing.T) { t.Run("relative glob", func(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test.txt"}, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test.txt"}, false) expected := []string{ "test_data/test_subfolder1/a.txt", "test_data/test_subfolder1/sub_sub_folder/b.txt", @@ -81,7 +81,7 @@ func TestGlobExcludes(t *testing.T) { }) t.Run("exact glob", func(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/*.txt"}, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/*.txt"}, false) expected := []string{ "test_data/test_subfolder1/a.txt", "test_data/test_subfolder1/sub_sub_folder/b.txt", @@ -91,7 +91,7 @@ func TestGlobExcludes(t *testing.T) { }) t.Run("entire directory", func(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/test_subfolder1/**"}, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/test_subfolder1/**"}, false) expected := []string{ "test_data/test_subfolder++/test.txt", "test_data/test.txt", @@ -100,7 +100,7 @@ func TestGlobExcludes(t *testing.T) { }) t.Run("entire directory via base path exclusion", func(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/test_subfolder1"}, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test_data/test_subfolder1"}, false) expected := []string{ "test_data/test_subfolder++/test.txt", "test_data/test.txt", @@ -109,7 +109,7 @@ func TestGlobExcludes(t *testing.T) { }) t.Run("mix of relative and total", func(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test.txt", "test_data/test_subfolder1/a.txt"}, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"test_data/**.txt"}, []string{"test.txt", "test_data/test_subfolder1/a.txt"}, false) expected := []string{ "test_data/test_subfolder1/sub_sub_folder/b.txt", } @@ -118,7 +118,7 @@ func TestGlobExcludes(t *testing.T) { } func TestCannotGlobBetweenPackageBoundaries(t *testing.T) { - files := Glob(buildFileNames, "src/fs", []string{"**/*.txt", "**/*.py"}, nil, false) + files := Glob(HostFS, buildFileNames, "src/fs", []string{"**/*.txt", "**/*.py"}, nil, false) expected := []string{ "test_data/test_subfolder++/test.txt", "test_data/test_subfolder1/a.txt", diff --git a/src/fs/iofs.go b/src/fs/iofs.go new file mode 100644 index 0000000000..2b3c253508 --- /dev/null +++ b/src/fs/iofs.go @@ -0,0 +1,19 @@ +package fs + +import ( + iofs "io/fs" + "os" +) + +type osFS struct{} + +func (osFS) ReadDir(name string) ([]iofs.DirEntry, error) { + return os.ReadDir(name) +} + +func (osFS) Open(name string) (iofs.File, error) { + return os.Open(name) +} + +// HostFS returns an io/fs.FS that behaves the same as the host OS i.e. the same way os.Open works. +var HostFS = osFS{} diff --git a/src/help/BUILD b/src/help/BUILD index 9778a99feb..3394be50a6 100644 --- a/src/help/BUILD +++ b/src/help/BUILD @@ -16,6 +16,7 @@ go_library( "//src/cli", "//src/cli/logging", "//src/core", + "//src/fs", "//src/parse/asp", "//src/plz", ], diff --git a/src/help/completion.go b/src/help/completion.go index df88cd3a1a..eee918b2ce 100644 --- a/src/help/completion.go +++ b/src/help/completion.go @@ -4,6 +4,7 @@ import ( "github.com/thought-machine/go-flags" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" ) // Topic is a help topic that implements completion for flags. @@ -18,7 +19,7 @@ func (topic *Topic) UnmarshalFlag(value string) error { // Complete implements the flags.Completer interface, which is used for shell completion. func (topic Topic) Complete(match string) []flags.Completion { - config, err := core.ReadDefaultConfigFiles(core.HostFS(), nil) + config, err := core.ReadDefaultConfigFiles(fs.HostFS, nil) if err != nil { config = core.DefaultConfiguration() } diff --git a/src/help/help.go b/src/help/help.go index 66ba6351da..32d32dcc68 100644 --- a/src/help/help.go +++ b/src/help/help.go @@ -13,6 +13,7 @@ import ( "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse/asp" "github.com/thought-machine/please/src/plz" ) @@ -28,7 +29,7 @@ const maxSuggestionDistance = 4 // Help prints help on a particular topic. // It returns true if the topic is known or false if it isn't. func Help(topic string) bool { - config, err := core.ReadDefaultConfigFiles(core.HostFS(), nil) + config, err := core.ReadDefaultConfigFiles(fs.HostFS, nil) if err != nil { // Don't bother the user if we can't load config files or whatever - just do our best. config = core.DefaultConfiguration() diff --git a/src/help/rules.go b/src/help/rules.go index 1e0b45a762..bf2babfff6 100644 --- a/src/help/rules.go +++ b/src/help/rules.go @@ -12,6 +12,7 @@ import ( "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse/asp" ) @@ -41,7 +42,7 @@ func PrintRuleArgs(files cli.StdinStrings) { func newState() *core.BuildState { // If we're in a repo, we might be able to read some stuff from there. if core.FindRepoRoot() { - if config, err := core.ReadDefaultConfigFiles(core.HostFS(), nil); err == nil { + if config, err := core.ReadDefaultConfigFiles(fs.HostFS, nil); err == nil { return core.NewBuildState(config) } } diff --git a/src/parse/asp/builtins.go b/src/parse/asp/builtins.go index 47e61e84ab..b31e43c5db 100644 --- a/src/parse/asp/builtins.go +++ b/src/parse/asp/builtins.go @@ -607,7 +607,7 @@ func glob(s *scope, args []pyObject) pyObject { allowEmpty := args[4].IsTruthy() exclude = append(exclude, s.state.Config.Parse.BuildFileName...) if s.globber == nil { - s.globber = fs.NewGlobber(s.state.Config.Parse.BuildFileName) + s.globber = fs.NewGlobber(fs.HostFS, s.state.Config.Parse.BuildFileName) } glob := s.globber.Glob(s.pkg.SourceRoot(), include, exclude, hidden, includeSymlinks) diff --git a/src/parse/asp/main/BUILD b/src/parse/asp/main/BUILD index dfd612763f..3911857723 100644 --- a/src/parse/asp/main/BUILD +++ b/src/parse/asp/main/BUILD @@ -7,6 +7,7 @@ go_binary( "//src/cli", "//src/cli/logging", "//src/core", + "//src/fs", "//src/parse/asp", ], ) diff --git a/src/parse/asp/main/main.go b/src/parse/asp/main/main.go index 62bfe4d317..6557c40a32 100644 --- a/src/parse/asp/main/main.go +++ b/src/parse/asp/main/main.go @@ -21,6 +21,7 @@ import ( "github.com/thought-machine/please/src/cli" "github.com/thought-machine/please/src/cli/logging" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse/asp" ) @@ -203,7 +204,7 @@ func main() { config := core.DefaultConfiguration() if !opts.NoConfig { var err error - config, err = core.ReadConfigFiles(core.HostFS(), []string{filepath.Join(core.MustFindRepoRoot(), core.ConfigFileName)}, nil) + config, err = core.ReadConfigFiles(fs.HostFS, []string{filepath.Join(core.MustFindRepoRoot(), core.ConfigFileName)}, nil) if err != nil { log.Fatalf("%s", err) } diff --git a/src/please.go b/src/please.go index 913fba5c95..3d6cf2b7f6 100644 --- a/src/please.go +++ b/src/please.go @@ -1013,7 +1013,7 @@ var buildFunctions = map[string]func() int{ // Check if tool is given as label or path and then run func runTool(_tool tool.Tool) int { c := core.DefaultConfiguration() - if cfg, err := core.ReadDefaultConfigFiles(core.HostFS(), opts.BuildFlags.Profile); err == nil { + if cfg, err := core.ReadDefaultConfigFiles(fs.HostFS, opts.BuildFlags.Profile); err == nil { c = cfg } t, _ := tool.MatchingTool(c, string(_tool)) @@ -1257,7 +1257,7 @@ func (l TargetsOrArgs) SeparateUnannotated() ([]core.BuildLabel, []string) { // readConfig reads the initial configuration files func readConfig() *core.Configuration { - cfg, err := core.ReadDefaultConfigFiles(core.HostFS(), opts.BuildFlags.Profile) + cfg, err := core.ReadDefaultConfigFiles(fs.HostFS, opts.BuildFlags.Profile) if err != nil { log.Fatalf("Error reading config file: %s", err) } else if err := cfg.ApplyOverrides(opts.BuildFlags.Option); err != nil { diff --git a/src/tool/tool_test.go b/src/tool/tool_test.go index 1595e91fe7..d98e182f52 100644 --- a/src/tool/tool_test.go +++ b/src/tool/tool_test.go @@ -6,10 +6,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" ) func TestMatchingTools(t *testing.T) { - c, err := core.ReadConfigFiles(core.HostFS(), nil, nil) + c, err := core.ReadConfigFiles(fs.HostFS, nil, nil) assert.NoError(t, err) assert.Equal(t, map[string]string{"langserver": "//_please:build_langserver"}, matchingTools(c, "la")) assert.Equal(t, map[string]string{"langserver": "//_please:build_langserver"}, matchingTools(c, "lang")) @@ -19,7 +20,7 @@ func TestMatchingTools(t *testing.T) { } func TestAllToolNames(t *testing.T) { - c, err := core.ReadConfigFiles(core.HostFS(), nil, nil) + c, err := core.ReadConfigFiles(fs.HostFS, nil, nil) assert.NoError(t, err) assert.Equal(t, []string{"javacworker"}, allToolNames(c, "ja")) } diff --git a/tools/build_langserver/lsp/BUILD b/tools/build_langserver/lsp/BUILD index 9424a305ba..b7c34291c1 100644 --- a/tools/build_langserver/lsp/BUILD +++ b/tools/build_langserver/lsp/BUILD @@ -16,6 +16,7 @@ go_library( "///third_party/go/gopkg.in_op_go-logging.v1//:go-logging.v1", "//rules", "//src/core", + "//src/fs", "//src/parse/asp", "//src/plz", "//tools/build_langserver/lsp/astutils", diff --git a/tools/build_langserver/lsp/lsp.go b/tools/build_langserver/lsp/lsp.go index 7d6c991891..0ed109eb80 100644 --- a/tools/build_langserver/lsp/lsp.go +++ b/tools/build_langserver/lsp/lsp.go @@ -19,6 +19,7 @@ import ( "github.com/thought-machine/please/rules" "github.com/thought-machine/please/src/core" + "github.com/thought-machine/please/src/fs" "github.com/thought-machine/please/src/parse/asp" "github.com/thought-machine/please/src/plz" ) @@ -187,7 +188,7 @@ func (h *Handler) initialize(params *lsp.InitializeParams) (*lsp.InitializeResul if err := os.Chdir(h.root); err != nil { return nil, err } - config, err := core.ReadDefaultConfigFiles(core.HostFS(), nil) + config, err := core.ReadDefaultConfigFiles(fs.HostFS, nil) if err != nil { log.Error("Error reading configuration: %s", err) config = core.DefaultConfiguration() diff --git a/tools/please_shim/main.go b/tools/please_shim/main.go index d5516a0f8d..7ce8577955 100644 --- a/tools/please_shim/main.go +++ b/tools/please_shim/main.go @@ -104,7 +104,7 @@ func findRootAndReadConfigFilesOnly() *core.Configuration { core.RepoRoot = abs } else if !core.FindRepoRoot() { log.Debug("Trying to find the default repo root on global config files") - if err := core.ReadDefaultGlobalConfigFilesOnly(core.HostFS(), config); err != nil { + if err := core.ReadDefaultGlobalConfigFilesOnly(fs.HostFS, config); err != nil { log.Fatalf("Error reading default global config file: %s", err) } // We are done here. There's no default repo root and we've read all @@ -116,7 +116,7 @@ func findRootAndReadConfigFilesOnly() *core.Configuration { } // At this point the repo root is known so read its config files. - if err := core.ReadDefaultConfigFilesOnly(core.HostFS(), config, opts.BuildFlags.Profile); err != nil { + if err := core.ReadDefaultConfigFilesOnly(fs.HostFS, config, opts.BuildFlags.Profile); err != nil { log.Fatalf("Error reading config file: %s", err) } @@ -165,7 +165,7 @@ func maybeUpdatePlease(state state, isUpdateCommand bool) { core.PleaseVersion = strings.TrimPrefix(strings.TrimSpace(string(out)), "Please version ") // Read and load the configuration as it would have been done by the main binary. - cfg, err := core.ReadDefaultConfigFiles(core.HostFS(), opts.BuildFlags.Profile) + cfg, err := core.ReadDefaultConfigFiles(fs.HostFS, opts.BuildFlags.Profile) if err != nil { log.Fatalf("Error reading config file: %s", err) } else if err := cfg.ApplyOverrides(opts.BuildFlags.Option); err != nil {