FEATURE: command to prebuild environments (v13.3.0)

- feature: command for prebuilding environments (from files or from URLs) - improvement: rcc version visible in "Toplevel" command list - added support for "cloud.ReadFile" functionality - bugfix: wrapped os.TempDir functionality to ensure directory exists
mikaukora · Jan 27, 2023 · a13e2f8 · a13e2f8
1 parent 6dd10a7
commit a13e2f8
Show file tree

Hide file tree

Showing 20 changed files with 172 additions and 20 deletions.
diff --git a/cloud/readfile.go b/cloud/readfile.go
@@ -0,0 +1,28 @@
+package cloud
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/robocorp/rcc/common"
+	"github.com/robocorp/rcc/pathlib"
+)
+
+func ReadFile(resource string) ([]byte, error) {
+	link, err := url.ParseRequestURI(resource)
+	if err != nil {
+		return os.ReadFile(resource)
+	}
+	if link.Scheme == "file" || link.Scheme == "" {
+		return os.ReadFile(link.Path)
+	}
+	tempfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("temp%x.part", common.When))
+	defer os.Remove(tempfile)
+	err = Download(resource, tempfile)
+	if err != nil {
+		return nil, err
+	}
+	return os.ReadFile(tempfile)
+}
diff --git a/cmd/assistantRun.go b/cmd/assistantRun.go
@@ -67,7 +67,7 @@ var assistantRunCmd = &cobra.Command{
 		common.Debug("Robot Assistant run-id is %v.", assistant.RunId)
 		common.Debug("With task '%v' from zip %v.", assistant.TaskName, assistant.Zipfile)
 		sentinelTime := time.Now()
-		workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", common.When))
+		workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When))
 		defer os.RemoveAll(workarea)
 		common.Debug("Using temporary workarea: %v", workarea)
 		reason = "UNZIP_FAILURE"

diff --git a/cmd/carrier.go b/cmd/carrier.go
@@ -48,7 +48,7 @@ func runCarrier() error {
 		return err
 	}
 	sentinelTime := time.Now()
-	workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", common.When))
+	workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When))
 	defer os.RemoveAll(workarea)
 	common.Debug("Using temporary workarea: %v", workarea)
 	carrier, err := operations.FindExecutable()

diff --git a/cmd/cloudPrepare.go b/cmd/cloudPrepare.go
@@ -27,10 +27,10 @@ var prepareCloudCmd = &cobra.Command{
 			defer common.Stopwatch("Cloud prepare lasted").Report()
 		}
 
-		zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("summon%x.zip", common.When))
+		zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("summon%x.zip", common.When))
 		defer os.Remove(zipfile)
 
-		workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", common.When))
+		workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When))
 		defer os.RemoveAll(workarea)
 
 		account := operations.AccountByName(AccountName())

diff --git a/cmd/communitypull.go b/cmd/communitypull.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 
 	"github.com/spf13/cobra"
@@ -26,7 +27,7 @@ var communityPullCmd = &cobra.Command{
 			defer common.Stopwatch("Pull lasted").Report()
 		}
 
-		zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("pull%x.zip", common.When))
+		zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("pull%x.zip", common.When))
 		defer os.Remove(zipfile)
 		common.Debug("Using temporary zipfile at %v", zipfile)
 

diff --git a/cmd/holotreeBootstrap.go b/cmd/holotreeBootstrap.go
@@ -8,6 +8,7 @@ import (
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/htfs"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 	"github.com/robocorp/rcc/robot"
 	"github.com/spf13/cobra"
@@ -21,7 +22,7 @@ func updateEnvironments(robots []string) {
 	tree, err := htfs.New()
 	pretty.Guard(err == nil, 2, "Holotree creation error: %v", err)
 	for at, template := range robots {
-		workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x%x", common.When, at))
+		workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x%x", common.When, at))
 		defer os.RemoveAll(workarea)
 		common.Debug("Using temporary workarea: %v", workarea)
 		err = operations.InitializeWorkarea(workarea, template, false, forceFlag)

diff --git a/cmd/holotreeImport.go b/cmd/holotreeImport.go
@@ -9,6 +9,7 @@ import (
 	"github.com/robocorp/rcc/cloud"
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 	"github.com/spf13/cobra"
 )
@@ -23,7 +24,7 @@ func isUrl(name string) bool {
 
 func temporaryDownload(at int, link string) (string, error) {
 	common.Timeline("Download %v", link)
-	zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("hololib%x%x.zip", common.When, at))
+	zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("hololib%x%x.zip", common.When, at))
 	err := cloud.Download(link, zipfile)
 	if err != nil {
 		return "", err

diff --git a/cmd/holotreePrebuild.go b/cmd/holotreePrebuild.go
@@ -0,0 +1,100 @@
+package cmd
+
+import (
+	"net/url"
+	"path/filepath"
+	"strings"
+
+	"github.com/robocorp/rcc/cloud"
+	"github.com/robocorp/rcc/common"
+	"github.com/robocorp/rcc/htfs"
+	"github.com/robocorp/rcc/pathlib"
+	"github.com/robocorp/rcc/pretty"
+	"github.com/spf13/cobra"
+)
+
+var (
+	metafileFlag bool
+)
+
+func conditionalExpand(filename string) string {
+	if !pathlib.IsFile(filename) {
+		return filename
+	}
+	fullpath, err := filepath.Abs(filename)
+	if err != nil {
+		return filename
+	}
+	return fullpath
+}
+
+func resolveMetafile(link string) ([]string, error) {
+	origin, err := url.Parse(link)
+	refok := err == nil
+	raw, err := cloud.ReadFile(link)
+	if err != nil {
+		return nil, err
+	}
+	result := []string{}
+	for _, line := range strings.SplitAfter(string(raw), "\n") {
+		flat := strings.TrimSpace(line)
+		if strings.HasPrefix(flat, "#") || len(flat) == 0 {
+			continue
+		}
+		here, err := url.Parse(flat)
+		if refok && err == nil {
+			relative := origin.ResolveReference(here)
+			result = append(result, relative.String())
+		} else {
+			result = append(result, flat)
+		}
+	}
+	return result, nil
+}
+
+func metafileExpansion(links []string, expand bool) []string {
+	if !expand {
+		return links
+	}
+	result := []string{}
+	for _, metalink := range links {
+		links, err := resolveMetafile(conditionalExpand(metalink))
+		if err != nil {
+			pretty.Warning("Failed to resolve %q metafile, reason: %v", metalink, err)
+			continue
+		}
+		result = append(result, links...)
+	}
+	return result
+}
+
+var holotreePrebuildCmd = &cobra.Command{
+	Use:   "prebuild",
+	Short: "Prebuild hololib from given set of environment descriptors.",
+	Long:  "Prebuild hololib from given set of environment descriptors.",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		if common.DebugFlag {
+			defer common.Stopwatch("Holotree prebuild lasted").Report()
+		}
+
+		total, failed := 0, 0
+
+		for _, configfile := range metafileExpansion(args, metafileFlag) {
+			total += 1
+			pretty.Note("Now building config %q", configfile)
+			_, _, err := htfs.NewEnvironment(configfile, "", false, false)
+			if err != nil {
+				failed += 1
+				pretty.Warning("Holotree recording error: %v", err)
+			}
+		}
+		pretty.Guard(failed == 0, 1, "%d out of %d environment builds failed! See output above for details.", failed, total)
+		pretty.Ok()
+	},
+}
+
+func init() {
+	holotreeCmd.AddCommand(holotreePrebuildCmd)
+	holotreePrebuildCmd.Flags().BoolVarP(&metafileFlag, "metafile", "m", false, "Input arguments are actually files containing links/filenames of environment descriptors.")
+}
diff --git a/cmd/peercc/main.go b/cmd/peercc/main.go
@@ -20,7 +20,7 @@ var (
 )
 
 func defaultHoldLocation() string {
-	where, err := pathlib.Abs(filepath.Join(os.TempDir(), "peercchold"))
+	where, err := pathlib.Abs(filepath.Join(pathlib.TempDir(), "peercchold"))
 	if err != nil {
 		return "temphold"
 	}

diff --git a/cmd/pull.go b/cmd/pull.go
@@ -8,6 +8,7 @@ import (
 	"github.com/robocorp/rcc/cloud"
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 
 	"github.com/spf13/cobra"
@@ -32,7 +33,7 @@ var pullCmd = &cobra.Command{
 			pretty.Exit(2, "Could not create client for endpoint: %v reason %v", account.Endpoint, err)
 		}
 
-		zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("pull%x.zip", common.When))
+		zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("pull%x.zip", common.When))
 		defer os.Remove(zipfile)
 		common.Debug("Using temporary zipfile at %v", zipfile)
 

diff --git a/cmd/push.go b/cmd/push.go
@@ -8,6 +8,7 @@ import (
 	"github.com/robocorp/rcc/cloud"
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/operations"
+	"github.com/robocorp/rcc/pathlib"
 	"github.com/robocorp/rcc/pretty"
 
 	"github.com/spf13/cobra"
@@ -30,7 +31,7 @@ var pushCmd = &cobra.Command{
 			pretty.Exit(2, "Could not create client for endpoint: %v reason: %v", account.Endpoint, err)
 		}
 
-		zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("push%x.zip", common.When))
+		zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("push%x.zip", common.When))
 		defer os.Remove(zipfile)
 		common.Debug("Using temporary zipfile at %v", zipfile)
 

diff --git a/cmd/root.go b/cmd/root.go
@@ -24,7 +24,7 @@ var (
 )
 
 func toplevelCommands(parent *cobra.Command) {
-	common.Log("\nToplevel commands")
+	common.Log("\nToplevel commands (%v)", common.Version)
 	for _, child := range parent.Commands() {
 		if child.Hidden || len(child.Commands()) > 0 {
 			continue

diff --git a/cmd/testrun.go b/cmd/testrun.go
@@ -28,7 +28,7 @@ var testrunCmd = &cobra.Command{
 			defer common.Stopwatch("Task testrun lasted").Report()
 		}
 		now := time.Now()
-		zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("testrun%x.zip", common.When))
+		zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("testrun%x.zip", common.When))
 		defer os.Remove(zipfile)
 		common.Debug("Using temporary zip file: %v", zipfile)
 		sourceDir := filepath.Dir(robotFile)
@@ -42,7 +42,7 @@ var testrunCmd = &cobra.Command{
 			pretty.Exit(2, "Error: %v", err)
 		}
 		sentinelTime := time.Now()
-		workarea := filepath.Join(os.TempDir(), fmt.Sprintf("workarea%x", common.When))
+		workarea := filepath.Join(pathlib.TempDir(), fmt.Sprintf("workarea%x", common.When))
 		defer os.RemoveAll(workarea)
 		common.Debug("Using temporary workarea: %v", workarea)
 		err = operations.Unzip(workarea, zipfile, false, true, true)

diff --git a/common/version.go b/common/version.go
@@ -1,5 +1,5 @@
 package common
 
 const (
-	Version = `v13.2.0`
+	Version = `v13.3.0`
 )
diff --git a/conda/condayaml.go b/conda/condayaml.go
@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/robocorp/rcc/cloud"
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/pathlib"
 
@@ -550,7 +551,7 @@ func CondaYamlFrom(content []byte) (*Environment, error) {
 }
 
 func ReadCondaYaml(filename string) (*Environment, error) {
-	content, err := os.ReadFile(filename)
+	content, err := cloud.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("%q: %w", filename, err)
 	}

diff --git a/conda/workflows.go b/conda/workflows.go
@@ -349,8 +349,8 @@ func LegacyEnvironment(force bool, configurations ...string) error {
 
 	freshInstall := true
 
-	condaYaml := filepath.Join(os.TempDir(), fmt.Sprintf("conda_%x.yaml", common.When))
-	requirementsText := filepath.Join(os.TempDir(), fmt.Sprintf("require_%x.txt", common.When))
+	condaYaml := filepath.Join(pathlib.TempDir(), fmt.Sprintf("conda_%x.yaml", common.When))
+	requirementsText := filepath.Join(pathlib.TempDir(), fmt.Sprintf("require_%x.txt", common.When))
 	common.Debug("Using temporary conda.yaml file: %v and requirement.txt file: %v", condaYaml, requirementsText)
 	key, yaml, finalEnv, err := temporaryConfig(condaYaml, requirementsText, true, configurations...)
 	if err != nil {

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -1,5 +1,12 @@
 # rcc change log
 
+## v13.3.0 (date: 27.1.2023)
+
+- feature: command for prebuilding environments (from files or from URLs)
+- improvement: rcc version visible in "Toplevel" command list
+- added support for "cloud.ReadFile" functionality
+- bugfix: wrapped os.TempDir functionality to ensure directory exists
+
 ## v13.2.0 (date: 24.1.2023)
 
 - feature: peercc force pulling holotree catalog from other remote peercc

diff --git a/operations/pull.go b/operations/pull.go
@@ -61,7 +61,7 @@ func downloadMissingEnvironmentParts(origin, catalogName, selection string) (fil
 	url := fmt.Sprintf("%s/delta/%s", origin, catalogName)
 
 	body := strings.NewReader(selection)
-	filename = filepath.Join(os.TempDir(), fmt.Sprintf("peercc_%x.zip", os.Getpid()))
+	filename = filepath.Join(pathlib.TempDir(), fmt.Sprintf("peercc_%x.zip", os.Getpid()))
 
 	client := &http.Client{Transport: settings.Global.ConfiguredHttpTransport()}
 	request, err := http.NewRequest("POST", url, body)
@@ -91,7 +91,7 @@ func downloadMissingEnvironmentParts(origin, catalogName, selection string) (fil
 	fail.On(err != nil, "Download failed, reason: %v", err)
 
 	sum := fmt.Sprintf("%02x", digest.Sum(nil))
-	finalname := filepath.Join(os.TempDir(), fmt.Sprintf("peercc_%s.zip", sum))
+	finalname := filepath.Join(pathlib.TempDir(), fmt.Sprintf("peercc_%s.zip", sum))
 	err = pathlib.TryRename("delta", filename, finalname)
 	fail.On(err != nil, "Rename %q -> %q failed, reason: %v", filename, finalname, err)
 

diff --git a/operations/updownload.go b/operations/updownload.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/robocorp/rcc/cloud"
 	"github.com/robocorp/rcc/common"
+	"github.com/robocorp/rcc/pathlib"
 )
 
 const (
@@ -168,7 +169,7 @@ func SummonRobotZipfile(client cloud.Client, account *account, workspaceId, robo
 	if ok {
 		return found, nil
 	}
-	zipfile := filepath.Join(os.TempDir(), fmt.Sprintf("summon%x.zip", common.When))
+	zipfile := filepath.Join(pathlib.TempDir(), fmt.Sprintf("summon%x.zip", common.When))
 	err := DownloadCommand(client, account, workspaceId, robotId, zipfile, common.DebugFlag)
 	if err != nil {
 		return "", err

diff --git a/pathlib/functions.go b/pathlib/functions.go
@@ -10,8 +10,18 @@ import (
 
 	"github.com/robocorp/rcc/common"
 	"github.com/robocorp/rcc/fail"
+	"github.com/robocorp/rcc/pretty"
 )
 
+func TempDir() string {
+	base := os.TempDir()
+	_, err := EnsureDirectory(base)
+	if err != nil {
+		pretty.Warning("TempDir %q challenge, reason: %v", base, err)
+	}
+	return base
+}
+
 func Exists(pathname string) bool {
 	_, err := os.Stat(pathname)
 	return !os.IsNotExist(err)