Merge pull request #11 from gammadia/multiple-steps

Allow multiple steps for jobs, fix error in template

client/jobfile/jobfile.go

@@ -16,7 +16,7 @@ type Jobfile struct {
 
 	Version  string
 	Name     string
-	Image    JobfileImage
+	Steps    []JobfileImage
 	Env      map[string]string
 	Services map[string]JobfileService
 	Tasks    []string
@@ -53,18 +53,20 @@ func (jobfile Jobfile) Validate() error {
 		return fmt.Errorf("name must be a valid identifier")
 	}
 
-	if jobfile.Image.Dockerfile == "" {
-		return fmt.Errorf("image.dockerfile is required")
-	}
-	if _, err := os.Stat(path.Join(jobfile.path, jobfile.Image.Dockerfile)); os.IsNotExist(err) {
-		return fmt.Errorf("image.dockerfile must be an existing file on disk")
-	}
+	for _, step := range jobfile.Steps {
+		if step.Dockerfile == "" {
+			return fmt.Errorf("image.dockerfile is required")
+		}
+		if _, err := os.Stat(path.Join(jobfile.path, step.Dockerfile)); os.IsNotExist(err) {
+			return fmt.Errorf("image.dockerfile must be an existing file on disk")
+		}
 
-	if jobfile.Image.Context == "" {
-		return fmt.Errorf("image.context is required")
-	}
-	if _, err := os.Stat(path.Join(jobfile.path, jobfile.Image.Context)); os.IsNotExist(err) {
-		return fmt.Errorf("image.context must be an existing folder on disk")
+		if step.Context == "" {
+			return fmt.Errorf("image.context is required")
+		}
+		if _, err := os.Stat(path.Join(jobfile.path, step.Context)); os.IsNotExist(err) {
+			return fmt.Errorf("image.context must be an existing folder on disk")
+		}
 	}
 
 	for key := range jobfile.Env {

client/jobfile/reader.go

@@ -56,23 +56,20 @@ func Read(file string, options ReadOptions) (job *proto.Job, err error) {
 		return nil, UnmarshalError{fmt.Errorf("validate: %w", err), source}
 	}
 
-	// Name
 	job.Name = jobfile.Name
-
-	// Tasks
 	job.Tasks = jobfile.Tasks
-
-	// Image
-	if job.Image, err = buildImage(
-		path.Join(workDir, jobfile.Image.Dockerfile),
-		path.Join(workDir, jobfile.Image.Context),
-		jobfile.Image.Options,
-		options,
-	); err != nil {
-		return nil, fmt.Errorf("build: %w", err)
+	job.Steps = make([]string, len(jobfile.Steps))
+	for i, step := range jobfile.Steps {
+		if job.Steps[i], err = buildImage(
+			path.Join(workDir, step.Dockerfile),
+			path.Join(workDir, step.Context),
+			step.Options,
+			options,
+		); err != nil {
+			return nil, fmt.Errorf("build (%d): %w", i+1, err)
+		}
 	}
 
-	// Env
 	if len(jobfile.Env) > 0 {
 		job.Env = lo.MapToSlice(jobfile.Env, func(key string, value string) *proto.Job_Env {
 			return &proto.Job_Env{
@@ -82,7 +79,6 @@ func Read(file string, options ReadOptions) (job *proto.Job, err error) {
 		})
 	}
 
-	// Services
 	for name, service := range jobfile.Services {
 		jobService := &proto.Job_Service{
 			Name:  name,
@@ -123,20 +119,25 @@ type TemplateData struct {
 }
 
 func evaluateTemplate(source string, dir string, options ReadOptions) (string, error) {
+	var userError error
 	tmpl, err := template.New("jobfile").Funcs(template.FuncMap{
 		"base64": func(s string) string {
 			return base64.StdEncoding.EncodeToString([]byte(s))
 		},
-		"error": func(err string) error {
-			return errors.New(err)
+		"error": func(err string) any {
+			userError = errors.New(err)
+			panic(nil)
 		},
-		"exec": func(args ...string) (string, error) {
+		"exec": func(args ...string) string {
 			cmd := exec.Command(args[0], args[1:]...)
 			cmd.Dir = dir
 			cmd.Stdin = os.Stdin
 			cmd.Stderr = os.Stderr
-			output, err := cmd.Output()
-			return strings.TrimRight(string(output), "\n\r"), err
+			if output, err := cmd.Output(); err != nil {
+				panic(err)
+			} else {
+				return strings.TrimRight(string(output), "\n\r")
+			}
 		},
 		"join": func(sep string, s []string) string {
 			return strings.Join(s, sep)
@@ -148,13 +149,16 @@ func evaluateTemplate(source string, dir string, options ReadOptions) (string, e
 		"lines": func(s string) []string {
 			return strings.Split(s, "\n")
 		},
-		"shell": func(script string) (string, error) {
+		"shell": func(script string) string {
 			cmd := exec.Command("/bin/sh", "-c", script)
 			cmd.Dir = dir
 			cmd.Stdin = os.Stdin
 			cmd.Stderr = os.Stderr
-			output, err := cmd.Output()
-			return strings.TrimRight(string(output), "\n\r"), err
+			if output, err := cmd.Output(); err != nil {
+				panic(err)
+			} else {
+				return strings.TrimRight(string(output), "\n\r")
+			}
 		},
 		"split": func(sep string, s string) []string {
 			return strings.Split(s, sep)
@@ -178,7 +182,7 @@ func evaluateTemplate(source string, dir string, options ReadOptions) (string, e
 
 	var output strings.Builder
 	if err := tmpl.Execute(&output, data); err != nil {
-		return "", fmt.Errorf("failed to execute template: %w", err)
+		return "", lo.Ternary(userError != nil, userError, err)
 	}
 
 	return output.String(), nil

client/jobfile/tests/jobfile/invalid_env_keys.yaml

@@ -1,7 +1,7 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 env:
   0/2: "foo"

client/jobfile/tests/jobfile/invalid_image_context.yaml

@@ -1,5 +1,5 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "./toto"
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "./toto"

client/jobfile/tests/jobfile/invalid_image_dockerfile.yaml

@@ -1,5 +1,5 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "toto.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "toto.dockerfile"
+    context: "."

client/jobfile/tests/jobfile/invalid_missing_image_context.yaml

@@ -1,4 +1,4 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
+steps:
+  - dockerfile: "a.dockerfile"

client/jobfile/tests/jobfile/invalid_missing_image_dockerfile.yaml

@@ -1,4 +1,4 @@
 version: "1"
 name: "test"
-image:
-  context: "."
+steps:
+  - context: "."

client/jobfile/tests/jobfile/invalid_missing_name.yaml

@@ -1,4 +1,4 @@
 version: "1"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."

client/jobfile/tests/jobfile/invalid_missing_services_health_cmd.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   mysql:
     image: "mysql"

client/jobfile/tests/jobfile/invalid_missing_services_image.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   mysql:
     env:

client/jobfile/tests/jobfile/invalid_name.yaml

@@ -1,5 +1,5 @@
 version: "1"
 name: "Toto ../"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."

client/jobfile/tests/jobfile/invalid_services_env_keys.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   mysql:
     image: "mysql"

client/jobfile/tests/jobfile/invalid_services_health_timeout.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   mysql:
     image: "mysql"

client/jobfile/tests/jobfile/invalid_services_keys.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   0/2:
     image: "mysql"

client/jobfile/tests/jobfile/invalid_services_map.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   - "mysql"
   - "redis"

client/jobfile/tests/jobfile/invalid_version.yaml

@@ -1,5 +1,5 @@
 version: "42"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."

client/jobfile/tests/jobfile/valid_full_featured.yaml

@@ -1,8 +1,8 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."
 services:
   redis:
     image: "redis:7.0.5-alpine"

client/jobfile/tests/jobfile/valid_minimalist.yaml

@@ -1,5 +1,5 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "a.dockerfile"
-  context: "."
+steps:
+  - dockerfile: "a.dockerfile"
+    context: "."

client/main.go

@@ -46,7 +46,7 @@ var alfredCmd = &cobra.Command{
 			port = "25373"
 		}
 		sshTunneling := lo.Must(cmd.Flags().GetBool("ssh-tunneling"))
-		if host == "127.0.0.1" && !cmd.Flags().Changed("ssh-tunneling") {
+		if (host == "127.0.0.1" || host == "localhost") && !cmd.Flags().Changed("ssh-tunneling") {
 			sshTunneling = false
 		}
 

client/run.go

@@ -52,13 +52,14 @@ var runCmd = &cobra.Command{
 			return yaml.NewEncoder(cmd.OutOrStdout()).Encode(j)
 		}
 
-		spinner = ui.NewSpinner("Uploading image to server")
-		if err = sendImageToServer(cmd, j.Image); err != nil {
-			spinner.Fail()
-			return fmt.Errorf("failed to send image to server: %w", err)
-		} else {
-			spinner.Success()
+		spinner = ui.NewSpinner("Uploading images to server")
+		for _, image := range j.Steps {
+			if err = sendImageToServer(cmd, image); err != nil {
+				spinner.Fail()
+				return fmt.Errorf("failed to send image '%s' to server: %w", image, err)
+			}
 		}
+		spinner.Success()
 
 		spinner = ui.NewSpinner("Scheduling job")
 		job, err := client.ScheduleJob(cmd.Context(), &proto.ScheduleJobRequest{Job: j})

hack/job.sh

@@ -3,4 +3,4 @@
 #mysql -u root -proot -h mysql mysql <<< 'SHOW DATABASES;'
 # ^- binary             ^- host  ^- database
 
-echo "Hello, $ALFRED_TASK" > $ALFRED_OUTPUT/hello.txt
+echo "Hello, $ALFRED_TASK" >> $ALFRED_OUTPUT/hello.txt

hack/job.yaml

@@ -1,8 +1,5 @@
 version: "1"
 name: "test"
-image:
-  dockerfile: "job.dockerfile"
-  context: "."
 services:
 #  redis:
 #    image: "redis:7.0.5-alpine"
@@ -20,8 +17,13 @@ services:
 #      timeout: "10s"
 #      interval: "2s"
 #      retries: 20
+steps:
+  - dockerfile: "job.dockerfile"
+    context: "."
+  - dockerfile: "job.dockerfile"
+    context: "."
 tasks:
-  {{range (lines (shell "sloth instance --app tipee list -o json | jq -r '.[] | select(.features.tests) | .id' | tail -n 2"))}}
+  {{range (lines (shell "sloth instance --app tipee list -o json | jq -r '.[] | select(.features.tests) | .id' | tail -n 1"))}}
   - {{.}}
   {{end}}