BUGFIX: ensure directories exists (v13.5.6)

- bugfix: create missing folders while creating and writing some files
- improvement: added optional top N biggest files sizes on catalog listing

cloud/client.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -226,8 +225,7 @@ func Download(url, filename string) error {
 		return fmt.Errorf("Downloading %q failed, reason: %q!", url, response.Status)
 	}
 
-	pathlib.EnsureDirectory(filepath.Dir(filename))
-	out, err := os.Create(filename)
+	out, err := pathlib.Create(filename)
 	if err != nil {
 		return err
 	}

cmd/holotreeCatalogs.go

@@ -5,18 +5,21 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/tabwriter"
 
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/htfs"
 	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
+	"github.com/robocorp/rcc/set"
 	"github.com/spf13/cobra"
 )
 
 var (
 	showIdentityYaml bool
+	topSizes         int
 )
 
 const mega = 1024 * 1024
@@ -67,7 +70,7 @@ func identityContentLines(catalog *htfs.Root) []string {
 	return result
 }
 
-func jsonCatalogDetails(roots []*htfs.Root) {
+func jsonCatalogDetails(roots []*htfs.Root, topN int) {
 	used := catalogUsedStats()
 	holder := make(map[string]map[string]interface{})
 	for _, catalog := range roots {
@@ -86,6 +89,9 @@ func jsonCatalogDetails(roots []*htfs.Root) {
 		if showIdentityYaml {
 			data["identity-content"] = identityContent(catalog)
 		}
+		if topN > 0 {
+			data[fmt.Sprintf("top%d", topN)] = catalog.Top(topN)
+		}
 		data["platform"] = catalog.Platform
 		data["directories"] = stats.Directories
 		data["files"] = stats.Files
@@ -100,7 +106,30 @@ func jsonCatalogDetails(roots []*htfs.Root) {
 	common.Stdout("%s\n", nice)
 }
 
-func listCatalogDetails(roots []*htfs.Root) {
+func percent(value, base float64) float64 {
+	if base == 0.0 {
+		return 0.0
+	}
+	return 100.0 * value / base
+}
+
+func dumpTopN(stats map[string]int64, total float64, tabbed *tabwriter.Writer) {
+	sizes := set.Values(stats)
+	sort.Slice(sizes, func(left, right int) bool {
+		return sizes[left] > sizes[right]
+	})
+	for _, focus := range sizes {
+		share := percent(float64(focus), total)
+		value, suffix := pathlib.HumaneSizer(focus)
+		for filename, size := range stats {
+			if focus == size {
+				tabbed.Write([]byte(fmt.Sprintf("\t\t\t%5.1f%%\t%6.1f%s\t%s\n", share, value, suffix, filename)))
+			}
+		}
+	}
+}
+
+func listCatalogDetails(roots []*htfs.Root, topN int) {
 	used := catalogUsedStats()
 	tabbed := tabwriter.NewWriter(os.Stderr, 2, 4, 2, ' ', 0)
 	tabbed.Write([]byte("Blueprint\tPlatform\tDirs  \tFiles  \tSize   \tidentity.yaml (gzipped blob inside hololib)\tHolotree path\tAge (days)\tIdle (days)\n"))
@@ -121,6 +150,9 @@ func listCatalogDetails(roots []*htfs.Root) {
 				tabbed.Write([]byte(fmt.Sprintf("\t\t\t\t\t%s\n", line)))
 			}
 		}
+		if topN > 0 {
+			dumpTopN(catalog.Top(topN), float64(stats.Bytes), tabbed)
+		}
 	}
 	tabbed.Flush()
 }
@@ -135,9 +167,9 @@ var holotreeCatalogsCmd = &cobra.Command{
 		}
 		_, roots := htfs.LoadCatalogs()
 		if jsonFlag {
-			jsonCatalogDetails(roots)
+			jsonCatalogDetails(roots, topSizes)
 		} else {
-			listCatalogDetails(roots)
+			listCatalogDetails(roots, topSizes)
 		}
 		pretty.Ok()
 	},
@@ -147,4 +179,5 @@ func init() {
 	holotreeCmd.AddCommand(holotreeCatalogsCmd)
 	holotreeCatalogsCmd.Flags().BoolVarP(&jsonFlag, "json", "j", false, "Output in JSON format")
 	holotreeCatalogsCmd.Flags().BoolVarP(&showIdentityYaml, "identity", "i", false, "Show identity.yaml in catalog context.")
+	holotreeCatalogsCmd.Flags().IntVarP(&topSizes, "top", "t", 0, "Show top N sized files from catalog")
 }

cmd/holotreeVariables.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -11,6 +10,7 @@ import (
 	"github.com/robocorp/rcc/htfs"
 	"github.com/robocorp/rcc/journal"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 	"github.com/robocorp/rcc/robot"
 	"github.com/spf13/cobra"
@@ -72,7 +72,7 @@ func holotreeExpandEnvironment(userFiles []string, packfile, environment, worksp
 	pretty.Guard(err == nil, 5, "%s", err)
 
 	condafile := filepath.Join(common.RobocorpTemp(), htfs.BlueprintHash(holotreeBlueprint))
-	err = os.WriteFile(condafile, holotreeBlueprint, 0o644)
+	err = pathlib.WriteFile(condafile, holotreeBlueprint, 0o644)
 	pretty.Guard(err == nil, 6, "%s", err)
 
 	holozip := ""

cmd/rcc/main.go

@@ -86,7 +86,7 @@ func markTempForRecycling() {
 	target := common.RobocorpTempName()
 	if pathlib.Exists(target) {
 		filename := filepath.Join(target, "recycle.now")
-		os.WriteFile(filename, []byte("True"), 0o644)
+		pathlib.WriteFile(filename, []byte("True"), 0o644)
 		common.Debug("Marked %q for recycling.", target)
 		markedAlready = true
 	}

cmd/root.go

@@ -124,7 +124,7 @@ func init() {
 func initConfig() {
 	if profilefile != "" {
 		common.TimelineBegin("profiling run started")
-		sink, err := os.Create(profilefile)
+		sink, err := pathlib.Create(profilefile)
 		pretty.Guard(err == nil, 5, "Failed to create profile file %q, reason %v.", profilefile, err)
 		err = pprof.StartCPUProfile(sink)
 		pretty.Guard(err == nil, 6, "Failed to start CPU profile, reason %v.", err)

cmd/speed.go

@@ -12,6 +12,7 @@ import (
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/htfs"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 
 	"github.com/spf13/cobra"
@@ -68,7 +69,7 @@ var speedtestCmd = &cobra.Command{
 			pretty.Exit(1, "Error: %v", err)
 		}
 		condafile := filepath.Join(folder, "speedtest.yaml")
-		err = os.WriteFile(condafile, content, 0o666)
+		err = pathlib.WriteFile(condafile, content, 0o666)
 		if err != nil {
 			pretty.Exit(2, "Error: %v", err)
 		}

common/version.go

@@ -1,5 +1,5 @@
 package common
 
 const (
-	Version = `v13.5.5`
+	Version = `v13.5.6`
 )

conda/activate.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/robocorp/rcc/common"
+	"github.com/robocorp/rcc/pathlib"
 )
 
 const (
@@ -62,7 +63,7 @@ func createScript(targetFolder string) (string, error) {
 	script.Execute(buffer, details)
 
 	scriptfile := filepath.Join(targetFolder, fmt.Sprintf("rcc_activate%s", commandSuffix))
-	err = os.WriteFile(scriptfile, buffer.Bytes(), 0o755)
+	err = pathlib.WriteFile(scriptfile, buffer.Bytes(), 0o755)
 	if err != nil {
 		return "", err
 	}
@@ -138,7 +139,7 @@ func Activate(sink io.Writer, targetFolder string) error {
 		return err
 	}
 	targetJson := filepath.Join(targetFolder, activateFile)
-	err = os.WriteFile(targetJson, body, 0o644)
+	err = pathlib.WriteFile(targetJson, body, 0o644)
 	if err != nil {
 		return err
 	}

conda/condayaml.go

@@ -430,13 +430,13 @@ func (it *Environment) SaveAs(filename string) error {
 		return err
 	}
 	common.Trace("FINAL conda environment file as %v:\n---\n%v---", filename, content)
-	return os.WriteFile(filename, []byte(content), 0o640)
+	return pathlib.WriteFile(filename, []byte(content), 0o640)
 }
 
 func (it *Environment) SaveAsRequirements(filename string) error {
 	content := it.AsRequirementsText()
 	common.Trace("FINAL pip requirements as %v:\n---\n%v\n---", filename, content)
-	return os.WriteFile(filename, []byte(content), 0o640)
+	return pathlib.WriteFile(filename, []byte(content), 0o640)
 }
 
 func (it *Environment) AsYaml() (string, error) {

conda/dependencies.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/fail"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 	"gopkg.in/yaml.v2"
 )
@@ -125,7 +126,7 @@ func goldenMaster(targetFolder string, pipUsed bool) (err error) {
 	fail.On(err != nil, "Failed to make yaml, reason: %v", err)
 	goldenfile := GoldenMasterFilename(targetFolder)
 	common.Debug("%sGolden EE file at: %v%s", pretty.Yellow, goldenfile, pretty.Reset)
-	return os.WriteFile(goldenfile, body, 0644)
+	return pathlib.WriteFile(goldenfile, body, 0644)
 }
 
 func LoadWantedDependencies(filename string) dependencies {

conda/workflows.go

@@ -272,7 +272,7 @@ func newLiveInternal(yaml, condaYaml, requirementsText, key string, force, fresh
 	common.Debug("===  finalize phase ===")
 
 	markerFile := filepath.Join(targetFolder, "identity.yaml")
-	err = os.WriteFile(markerFile, []byte(yaml), 0o644)
+	err = pathlib.WriteFile(markerFile, []byte(yaml), 0o644)
 	if err != nil {
 		return false, false
 	}
@@ -281,7 +281,7 @@ func newLiveInternal(yaml, condaYaml, requirementsText, key string, force, fresh
 	if ok {
 		venvContent := fmt.Sprintf(venvTemplate, targetFolder, pythonVersionAt(targetFolder))
 		venvFile := filepath.Join(targetFolder, "pyvenv.cfg")
-		err = os.WriteFile(venvFile, []byte(venvContent), 0o644)
+		err = pathlib.WriteFile(venvFile, []byte(venvContent), 0o644)
 		if err != nil {
 			return false, false
 		}

docs/changelog.md

@@ -1,5 +1,10 @@
 # rcc change log
 
+## v13.5.6 (date: 8.2.2023)
+
+- bugfix: create missing folders while creating and writing some files
+- improvement: added optional top N biggest files sizes on catalog listing
+
 ## v13.5.5 (date: 7.2.2023)
 
 - bugfix: contain output of checking long path support on Windows

htfs/directory.go

@@ -15,6 +15,7 @@ import (
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/fail"
 	"github.com/robocorp/rcc/pathlib"
+	"github.com/robocorp/rcc/set"
 )
 
 var (
@@ -72,6 +73,24 @@ func NewRoot(path string) (*Root, error) {
 	}, nil
 }
 
+func (it *Root) Top(count int) map[string]int64 {
+	target := make(map[string]int64)
+	it.Tree.fillSizes("", target)
+	sizes := set.Values(target)
+	total := len(sizes)
+	if total > count {
+		sizes = sizes[total-count:]
+	}
+	members := set.Membership(sizes)
+	result := make(map[string]int64)
+	for filename, size := range target {
+		if members[size] {
+			result[filename] = size
+		}
+	}
+	return result
+}
+
 func (it *Root) Show(filename string) ([]byte, error) {
 	return it.Tree.Show(filepath.SplitList(filename), filename)
 }
@@ -163,7 +182,7 @@ func (it *Root) SaveAs(filename string) error {
 	if err != nil {
 		return err
 	}
-	sink, err := os.Create(filename)
+	sink, err := pathlib.Create(filename)
 	if err != nil {
 		return err
 	}
@@ -225,6 +244,15 @@ func showFile(filename string) (content []byte, err error) {
 	return sink.Bytes(), nil
 }
 
+func (it *Dir) fillSizes(prefix string, target map[string]int64) {
+	for filename, file := range it.Files {
+		target[filepath.Join(prefix, filename)] = file.Size
+	}
+	for dirname, dir := range it.Dirs {
+		dir.fillSizes(filepath.Join(prefix, dirname), target)
+	}
+}
+
 func (it *Dir) Show(path []string, fullpath string) ([]byte, error) {
 	if len(path) > 1 {
 		subtree, ok := it.Dirs[path[0]]

htfs/library.go

@@ -165,7 +165,7 @@ func (it *hololib) Export(catalogs, known []string, archive string) (err error)
 	common.TimelineBegin("holotree export start")
 	defer common.TimelineEnd()
 
-	handle, err := os.Create(archive)
+	handle, err := pathlib.Create(archive)
 	fail.On(err != nil, "Could not create archive %q.", archive)
 	writer := zip.NewWriter(handle)
 	defer writer.Close()

operations/cache.go

@@ -96,7 +96,7 @@ func (it *Cache) Save() error {
 	}
 	defer locker.Release()
 
-	sink, err := os.Create(cacheLocation())
+	sink, err := pathlib.Create(cacheLocation())
 	if err != nil {
 		return err
 	}

operations/community.go

@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"regexp"
 	"strings"
 
 	"github.com/robocorp/rcc/common"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/settings"
 )
 
@@ -58,7 +58,7 @@ func DownloadCommunityRobot(url, filename string) error {
 		return fmt.Errorf("%s (%s)", response.Status, url)
 	}
 
-	out, err := os.Create(filename)
+	out, err := pathlib.Create(filename)
 	if err != nil {
 		return err
 	}

operations/fixing.go

@@ -44,7 +44,7 @@ func fixShellFile(fullpath string) {
 		return
 	}
 	common.Debug("Fixing newlines in file: %v", fullpath)
-	err = os.WriteFile(fullpath, ToUnix(content), 0o755)
+	err = pathlib.WriteFile(fullpath, ToUnix(content), 0o755)
 	if err != nil {
 		common.Log("Failure %v while fixing newlines in %v!", err, fullpath)
 	}

operations/pull.go

@@ -89,7 +89,7 @@ func downloadMissingEnvironmentParts(count int, origin, catalogName, selection s
 
 	fail.On(response.StatusCode < 200 || 299 < response.StatusCode, "%s (%s)", response.Status, url)
 
-	out, err := os.Create(filename)
+	out, err := pathlib.Create(filename)
 	fail.On(err != nil, "Creating temporary file %q failed, reason: %v", filename, err)
 	defer pathlib.TryRemove("temporary", filename)
 

operations/updownload.go

@@ -98,7 +98,7 @@ func putContent(client cloud.Client, awsUrl, zipfile string) error {
 }
 
 func getContent(client cloud.Client, awsUrl, zipfile string) error {
-	handle, err := os.Create(zipfile)
+	handle, err := pathlib.Create(zipfile)
 	if err != nil {
 		return err
 	}