BUGFIX: holotree delete slowness (v13.5.7)

- bugfix: holotree delete and plan were doing too many calls to find same
  environments (which mean they were really slow)
- some name refactorings to clarify intent of functions

cmd/holotreeDelete.go

@@ -9,15 +9,14 @@ import (
 )
 
 func deleteByPartialIdentity(partials []string) {
-	for _, prefix := range partials {
-		for _, label := range htfs.FindEnvironment(prefix) {
-			common.Log("Removing %v", label)
-			if dryFlag {
-				continue
-			}
-			err := htfs.RemoveHolotreeSpace(label)
-			pretty.Guard(err == nil, 1, "Error: %v", err)
+	_, roots := htfs.LoadCatalogs()
+	for _, label := range roots.FindEnvironments(partials) {
+		common.Log("Removing %v", label)
+		if dryFlag {
+			continue
 		}
+		err := roots.RemoveHolotreeSpace(label)
+		pretty.Guard(err == nil, 1, "Error: %v", err)
 	}
 }
 

cmd/holotreeExport.go

@@ -36,20 +36,20 @@ func holotreeExport(catalogs, known []string, archive string) {
 
 func listCatalogs(jsonForm bool) {
 	if jsonForm {
-		nice, err := json.MarshalIndent(htfs.Catalogs(), "", "  ")
+		nice, err := json.MarshalIndent(htfs.CatalogNames(), "", "  ")
 		pretty.Guard(err == nil, 2, "%s", err)
 		common.Stdout("%s\n", nice)
 	} else {
 		common.Log("Selectable catalogs (you can use substrings):")
-		for _, catalog := range htfs.Catalogs() {
+		for _, catalog := range htfs.CatalogNames() {
 			common.Log("- %s", catalog)
 		}
 	}
 }
 
 func selectCatalogs(filters []string) []string {
 	result := make([]string, 0, len(filters))
-	for _, catalog := range htfs.Catalogs() {
+	for _, catalog := range htfs.CatalogNames() {
 		for _, filter := range filters {
 			if strings.Contains(catalog, filter) {
 				result = append(result, catalog)

cmd/holotreeList.go

@@ -17,7 +17,8 @@ func humaneHolotreeSpaceListing() {
 	tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0)
 	tabbed.Write([]byte("Identity\tController\tSpace\tBlueprint\tFull path\n"))
 	tabbed.Write([]byte("--------\t----------\t-----\t--------\t---------\n"))
-	for _, space := range htfs.Spaces() {
+	_, roots := htfs.LoadCatalogs()
+	for _, space := range roots.Spaces() {
 		data := fmt.Sprintf("%s\t%s\t%s\t%s\t%s\n", space.Identity, space.Controller, space.Space, space.Blueprint, space.Path)
 		tabbed.Write([]byte(data))
 	}
@@ -26,7 +27,8 @@ func humaneHolotreeSpaceListing() {
 
 func jsonicHolotreeSpaceListing() {
 	details := make(map[string]map[string]string)
-	for _, space := range htfs.Spaces() {
+	_, roots := htfs.LoadCatalogs()
+	for _, space := range roots.Spaces() {
 		hold, ok := details[space.Identity]
 		if !ok {
 			hold = make(map[string]string)

cmd/holotreePlan.go

@@ -19,19 +19,18 @@ var holotreePlanCmd = &cobra.Command{
 
 	Run: func(cmd *cobra.Command, args []string) {
 		found := false
-		for _, prefix := range args {
-			for _, label := range htfs.FindEnvironment(prefix) {
-				planfile, ok := htfs.InstallationPlan(label)
-				pretty.Guard(ok, 1, "Could not find plan for: %v", label)
-				source, err := os.Open(planfile)
-				pretty.Guard(err == nil, 2, "Could not read plan %q, reason: %v", planfile, err)
-				defer source.Close()
-				analyzer := conda.NewPlanAnalyzer(false)
-				defer analyzer.Close()
-				sink := io.MultiWriter(os.Stdout, analyzer)
-				io.Copy(sink, source)
-				found = true
-			}
+		_, roots := htfs.LoadCatalogs()
+		for _, label := range roots.FindEnvironments(args) {
+			planfile, ok := roots.InstallationPlan(label)
+			pretty.Guard(ok, 1, "Could not find plan for: %v", label)
+			source, err := os.Open(planfile)
+			pretty.Guard(err == nil, 2, "Could not read plan %q, reason: %v", planfile, err)
+			defer source.Close()
+			analyzer := conda.NewPlanAnalyzer(false)
+			defer analyzer.Close()
+			sink := io.MultiWriter(os.Stdout, analyzer)
+			io.Copy(sink, source)
+			found = true
 		}
 		pretty.Guard(found, 3, "Nothing matched given plans!")
 		pretty.Ok()

common/version.go

@@ -1,5 +1,5 @@
 package common
 
 const (
-	Version = `v13.5.6`
+	Version = `v13.5.7`
 )

docs/changelog.md

@@ -1,5 +1,11 @@
 # rcc change log
 
+## v13.5.7 (date: 10.2.2023)
+
+- bugfix: holotree delete and plan were doing too many calls to find same
+  environments (which mean they were really slow)
+- some name refactorings to clarify intent of functions
+
 ## v13.5.6 (date: 8.2.2023)
 
 - bugfix: create missing folders while creating and writing some files

htfs/commands.go

@@ -168,38 +168,6 @@ func RecordEnvironment(tree MutableLibrary, blueprint []byte, force bool, scorec
 	return nil
 }
 
-func FindEnvironment(fragment string) []string {
-	result := make([]string, 0, 10)
-	for directory, _ := range Spacemap() {
-		name := filepath.Base(directory)
-		if strings.Contains(name, fragment) {
-			result = append(result, name)
-		}
-	}
-	return result
-}
-
-func InstallationPlan(hash string) (string, bool) {
-	finalplan := filepath.Join(common.HolotreeLocation(), hash, "rcc_plan.log")
-	return finalplan, pathlib.IsFile(finalplan)
-}
-
-func RemoveHolotreeSpace(label string) (err error) {
-	defer fail.Around(&err)
-
-	for directory, metafile := range Spacemap() {
-		name := filepath.Base(directory)
-		if name != label {
-			continue
-		}
-		pathlib.TryRemove("metafile", metafile)
-		pathlib.TryRemove("lockfile", directory+".lck")
-		err = pathlib.TryRemoveAll("space", directory)
-		fail.On(err != nil, "Problem removing %q, reason: %s.", directory, err)
-	}
-	return nil
-}
-
 func RobotBlueprints(userBlueprints []string, packfile string) (robot.Robot, []string) {
 	var err error
 	var config robot.Robot

htfs/directory.go

@@ -56,6 +56,81 @@ type Root struct {
 	source     string
 }
 
+type Roots []*Root
+
+func (it Roots) BaseFolders() []string {
+	result := []string{}
+	for _, root := range it {
+		result = append(result, filepath.Dir(root.Path))
+	}
+	return set.Set(result)
+}
+
+func (it Roots) Spaces() Roots {
+	roots := make(Roots, 0, 20)
+	for directory, metafile := range it.Spacemap() {
+		root, err := NewRoot(directory)
+		if err != nil {
+			continue
+		}
+		err = root.LoadFrom(metafile)
+		if err != nil {
+			continue
+		}
+		roots = append(roots, root)
+	}
+	return roots
+}
+
+func (it Roots) Spacemap() map[string]string {
+	result := make(map[string]string)
+	for _, basedir := range it.BaseFolders() {
+		for _, metafile := range pathlib.Glob(basedir, "*.meta") {
+			result[metafile[:len(metafile)-5]] = metafile
+		}
+	}
+	return result
+}
+
+func (it Roots) FindEnvironments(fragments []string) []string {
+	result := make([]string, 0, 10)
+	for directory, _ := range it.Spacemap() {
+		name := filepath.Base(directory)
+		for _, fragment := range fragments {
+			if strings.Contains(name, fragment) {
+				result = append(result, name)
+			}
+		}
+	}
+	return set.Set(result)
+}
+
+func (it Roots) InstallationPlan(hash string) (string, bool) {
+	for _, directory := range it.BaseFolders() {
+		finalplan := filepath.Join(directory, hash, "rcc_plan.log")
+		if pathlib.IsFile(finalplan) {
+			return finalplan, true
+		}
+	}
+	return "", false
+}
+
+func (it Roots) RemoveHolotreeSpace(label string) (err error) {
+	defer fail.Around(&err)
+
+	for directory, metafile := range it.Spacemap() {
+		name := filepath.Base(directory)
+		if name != label {
+			continue
+		}
+		pathlib.TryRemove("metafile", metafile)
+		pathlib.TryRemove("lockfile", directory+".lck")
+		err = pathlib.TryRemoveAll("space", directory)
+		fail.On(err != nil, "Problem removing %q, reason: %s.", directory, err)
+	}
+	return nil
+}
+
 func NewRoot(path string) (*Root, error) {
 	fullpath, err := pathlib.Abs(path)
 	if err != nil {
@@ -206,8 +281,6 @@ func (it *Root) ReadFrom(source io.Reader) error {
 }
 
 func (it *Root) LoadFrom(filename string) error {
-	common.TimelineBegin("holotree load %q", filename)
-	defer common.TimelineEnd()
 	source, err := os.Open(filename)
 	if err != nil {
 		return err
@@ -218,6 +291,7 @@ func (it *Root) LoadFrom(filename string) error {
 		return err
 	}
 	it.source = filename
+	defer common.Timeline("holotree catalog %q loaded", filename)
 	defer reader.Close()
 	return it.ReadFrom(reader)
 }

htfs/functions.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"sync"
 
 	"github.com/robocorp/rcc/anywork"
@@ -504,8 +503,8 @@ func DigestLoader(root *Root, at int, slots []map[string]string) anywork.Work {
 	}
 }
 
-func ignoreFailedCatalogs(suspects []*Root) []*Root {
-	roots := make([]*Root, 0, len(suspects))
+func ignoreFailedCatalogs(suspects Roots) Roots {
+	roots := make(Roots, 0, len(suspects))
 	for _, root := range suspects {
 		if root != nil {
 			roots = append(roots, root)
@@ -514,11 +513,11 @@ func ignoreFailedCatalogs(suspects []*Root) []*Root {
 	return roots
 }
 
-func LoadCatalogs() ([]string, []*Root) {
+func LoadCatalogs() ([]string, Roots) {
 	common.TimelineBegin("catalog load start")
 	defer common.TimelineEnd()
-	catalogs := Catalogs()
-	roots := make([]*Root, len(catalogs))
+	catalogs := CatalogNames()
+	roots := make(Roots, len(catalogs))
 	for at, catalog := range catalogs {
 		fullpath := filepath.Join(common.HololibCatalogLocation(), catalog)
 		anywork.Backlog(CatalogLoader(fullpath, at, roots))
@@ -529,21 +528,7 @@ func LoadCatalogs() ([]string, []*Root) {
 	return catalogs, ignoreFailedCatalogs(roots)
 }
 
-func BaseFolders() []string {
-	_, roots := LoadCatalogs()
-	folders := make(map[string]bool)
-	result := []string{}
-	for _, root := range roots {
-		folders[filepath.Dir(root.Path)] = true
-	}
-	for folder, _ := range folders {
-		result = append(result, folder)
-	}
-	sort.Strings(result)
-	return result
-}
-
-func CatalogLoader(catalog string, at int, roots []*Root) anywork.Work {
+func CatalogLoader(catalog string, at int, roots Roots) anywork.Work {
 	return func() {
 		tempdir := filepath.Join(common.RobocorpTemp(), "shadow")
 		shadow, err := NewRoot(tempdir)

htfs/library.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -18,6 +17,7 @@ import (
 	"github.com/robocorp/rcc/journal"
 	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
+	"github.com/robocorp/rcc/set"
 )
 
 const (
@@ -290,40 +290,12 @@ func (it *hololib) queryBlueprint(key string) bool {
 	return pathlib.IsFile(catalog)
 }
 
-func Catalogs() []string {
+func CatalogNames() []string {
 	result := make([]string, 0, 10)
 	for _, catalog := range pathlib.Glob(common.HololibCatalogLocation(), "[0-9a-f]*v12.*") {
-		result = append(result, catalog)
+		result = append(result, filepath.Base(catalog))
 	}
-	sort.Strings(result)
-	return result
-}
-
-func Spacemap() map[string]string {
-	result := make(map[string]string)
-	for _, basedir := range BaseFolders() {
-		for _, metafile := range pathlib.Glob(basedir, "*.meta") {
-			fullpath := filepath.Join(basedir, metafile)
-			result[fullpath[:len(fullpath)-5]] = fullpath
-		}
-	}
-	return result
-}
-
-func Spaces() []*Root {
-	roots := make([]*Root, 0, 20)
-	for directory, metafile := range Spacemap() {
-		root, err := NewRoot(directory)
-		if err != nil {
-			continue
-		}
-		err = root.LoadFrom(metafile)
-		if err != nil {
-			continue
-		}
-		roots = append(roots, root)
-	}
-	return roots
+	return set.Set(result)
 }
 
 func ControllerSpaceName(client, tag []byte) string {

operations/diagnostics.go

@@ -542,10 +542,10 @@ func diagnoseFilesUnmarshal(tool Unmarshaler, label, rootdir string, paths []str
 }
 
 func addFileDiagnostics(rootdir string, target *common.DiagnosticStatus) {
-	jsons := pathlib.Glob(rootdir, "*.json")
+	jsons := pathlib.RecursiveGlob(rootdir, "*.json")
 	diagnoseFilesUnmarshal(json.Unmarshal, "JSON", rootdir, jsons, target)
-	yamls := pathlib.Glob(rootdir, "*.yaml")
-	yamls = append(yamls, pathlib.Glob(rootdir, "*.yml")...)
+	yamls := pathlib.RecursiveGlob(rootdir, "*.yaml")
+	yamls = append(yamls, pathlib.RecursiveGlob(rootdir, "*.yml")...)
 	diagnoseFilesUnmarshal(yaml.Unmarshal, "YAML", rootdir, yamls, target)
 }
 

pathlib/functions.go

@@ -38,6 +38,12 @@ func WriteFile(filename string, data []byte, mode os.FileMode) error {
 	return os.WriteFile(filename, data, mode)
 }
 
+func Glob(directory string, pattern string) []string {
+	fullpath := filepath.Join(directory, pattern)
+	result, _ := filepath.Glob(fullpath)
+	return result
+}
+
 func Exists(pathname string) bool {
 	_, err := os.Stat(pathname)
 	return !os.IsNotExist(err)