Add support for generating go module deps

This adds a new `Generate` field on `Source` which is a list of
generator configs.
Today the only type of generator we support is for go modules but we'll
likely need to add support for other things, e.g. Rust cargo deps.

Multiple geneator configs are allowed in the case of a source with
multiple go modules defined in it.

Signed-off-by: Brian Goff <[email protected]>

docs/examples/go-md2man-1.yml

@@ -1,6 +1,7 @@
 # syntax=ghcr.io/azure/dalec/frontend:latest
 name: go-md2man
 version: 2.0.3
+revision: "1"
 packager: Dalec Example
 vendor: Dalec Example
 license: MIT

docs/examples/go-md2man-2.yml

@@ -1,6 +1,7 @@
 # syntax=ghcr.io/azure/dalec/frontend:latest
 name: go-md2man
 version: 2.0.3
+revision: "1"
 packager: Dalec Example
 vendor: Dalec Example
 license: MIT
@@ -9,27 +10,11 @@ website: https://github.com/cpuguy83/go-md2man
 
 sources:
   src:
+    generate:
+      - gomod: {}
     git:
       url: https://github.com/cpuguy83/go-md2man.git
       commit: "v2.0.3"
-  gomods: # This is required when the build environment does not allow network access. This downloads all the go modules.
-    path: /build/gomodcache # This is the path we will be extracing after running the command below.
-    image:
-      ref: mcr.microsoft.com/oss/go/microsoft/golang:1.21
-      cmd:
-        dir: /build/src
-        mounts:
-          # Mount a source (inline, under `spec`), so our command has access to it.
-          - dest: /build/src
-            spec:
-              git:
-                url: https://github.com/cpuguy83/go-md2man.git
-                commit: "v2.0.3"
-        steps:
-          - command: go mod download
-            env:
-              # This variable controls where the go modules are downloaded to.
-              GOMODCACHE: /build/gomodcache
 
 dependencies:
   build:
@@ -40,7 +25,6 @@ build:
     CGO_ENABLED: "0"
   steps:
     - command: |
-        export GOMODCACHE="$(pwd)/gomods"
         cd src
         go build -o go-md2man .
 

docs/spec.schema.json

@@ -291,6 +291,12 @@
 			],
 			"description": "Frontend encapsulates the configuration for a frontend to forward a build target to."
 		},
+		"GeneratorGomod": {
+			"properties": {},
+			"additionalProperties": false,
+			"type": "object",
+			"description": "GeneratorGomod is used to generate a go module cache from go module sources"
+		},
 		"ImageConfig": {
 			"properties": {
 				"entrypoint": {
@@ -466,6 +472,13 @@
 					},
 					"type": "array",
 					"description": "Excludes is a list of paths underneath `Path` to exclude, everything else is included"
+				},
+				"generate": {
+					"items": {
+						"$ref": "#/$defs/SourceGenerator"
+					},
+					"type": "array",
+					"description": "Generate is the list generators to run on the source.\n\nGenerators are used to generate additional sources from this source.\nAs an example the `godmod` generator can be used to generate a go module cache from a go source.\nHow a genator operates is dependent on the actual generator.\nGeneators may also cauuse modifications to the build environment.\n\nCurrently only one generator is supported: \"gomod\""
 				}
 			},
 			"additionalProperties": false,
@@ -524,6 +537,24 @@
 				"ref"
 			]
 		},
+		"SourceGenerator": {
+			"properties": {
+				"subpath": {
+					"type": "string",
+					"description": "Subpath is the path inside a source to run the generator from."
+				},
+				"gomod": {
+					"$ref": "#/$defs/GeneratorGomod",
+					"description": "Gomod is the go module generator."
+				}
+			},
+			"additionalProperties": false,
+			"type": "object",
+			"required": [
+				"gomod"
+			],
+			"description": "SourceGenerator holds the configuration for a source generator."
+		},
 		"SourceGit": {
 			"properties": {
 				"url": {

frontend/debug/handle_gomod.go

@@ -0,0 +1,60 @@
+package debug
+
+import (
+	"context"
+
+	"github.com/Azure/dalec"
+	"github.com/Azure/dalec/frontend"
+	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/gateway/client"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const keyGomodWorker = "context:gomod-worker"
+
+// Gomods outputs all the gomodule dependencies for the spec
+func Gomods(ctx context.Context, client gwclient.Client) (*client.Result, error) {
+	return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
+		sOpt, err := frontend.SourceOptFromClient(ctx, client)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		inputs, err := client.Inputs(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// Allow the client to override the worker image
+		// This is useful for keeping pre-built worker image, especially for CI.
+		worker, ok := inputs[keyGomodWorker]
+		if !ok {
+			worker = llb.Image("alpine:latest", llb.WithMetaResolver(client)).
+				Run(llb.Shlex("apk add --no-cache go git ca-certificates patch")).Root()
+		}
+
+		st, err := spec.GomodDeps(sOpt, worker)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		def, err := st.Marshal(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		res, err := client.Solve(ctx, gwclient.SolveRequest{
+			Definition: def.ToPB(),
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+
+		ref, err := res.SingleRef()
+		if err != nil {
+			return nil, nil, err
+		}
+		return ref, nil, nil
+	})
+}

frontend/debug/handler.go

@@ -21,6 +21,10 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		Name:        "sources",
 		Description: "Outputs all sources from a dalec spec file.",
 	})
+	r.Add("gomods", Gomods, &targets.Target{
+		Name:        "gomods",
+		Description: "Outputs all the gomodule dependencies for the spec",
+	})
 
 	return r.Handle(ctx, client)
 }

frontend/mariner2/handle_rpm.go

@@ -76,7 +76,7 @@ func getWorkerImage(resolver llb.ImageMetaResolver, opts ...llb.ConstraintsOpt)
 	opts = append(opts, dalec.ProgressGroup("Prepare worker image"))
 	return llb.Image(marinerRef, llb.WithMetaResolver(resolver), dalec.WithConstraints(opts...)).
 		Run(
-			shArgs("tdnf install -y rpm-build mariner-rpm-macros build-essential"),
+			shArgs("tdnf install -y rpm-build mariner-rpm-macros build-essential ca-certificates"),
 			defaultTdnfCacheMount(),
 			dalec.WithConstraints(opts...),
 		).
@@ -89,8 +89,8 @@ func installBuildDeps(spec *dalec.Spec, targetKey string, opts ...llb.Constraint
 		if len(deps) == 0 {
 			return in
 		}
-		opts = append(opts, dalec.ProgressGroup("Install build deps"))
 
+		opts = append(opts, dalec.ProgressGroup("Install build deps"))
 		return in.
 			Run(
 				shArgs(fmt.Sprintf("tdnf install --releasever=2.0 -y %s", strings.Join(deps, " "))),
@@ -102,12 +102,16 @@ func installBuildDeps(spec *dalec.Spec, targetKey string, opts ...llb.Constraint
 }
 
 func specToRpmLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
-	br, err := rpm.SpecToBuildrootLLB(spec, sOpt, targetKey, opts...)
+	base, err := getSpecWorker(sOpt.Resolver, spec, targetKey, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
+	br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}
 	specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")
 
-	base := getWorkerImage(sOpt.Resolver, opts...).With(installBuildDeps(spec, targetKey, opts...))
-	return rpm.Build(br, base, specPath, opts...), nil
+	return rpm.Build(br, base.With(installBuildDeps(spec, targetKey, opts...)), specPath, opts...), nil
 }

frontend/mariner2/handler.go

@@ -2,9 +2,13 @@ package mariner2
 
 import (
 	"context"
+	"errors"
+	"slices"
 
+	"github.com/Azure/dalec"
 	"github.com/Azure/dalec/frontend"
 	"github.com/Azure/dalec/frontend/rpm"
+	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	bktargets "github.com/moby/buildkit/frontend/subrequests/targets"
 )
@@ -20,7 +24,8 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		Name:        "rpm",
 		Description: "Builds an rpm and src.rpm for mariner2.",
 	})
-	mux.Add("rpm/debug", rpm.HandleDebug(), nil)
+
+	mux.Add("rpm/debug", handleDebug, nil)
 
 	mux.Add("container", handleContainer, &bktargets.Target{
 		Name:        "container",
@@ -35,3 +40,23 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 
 	return mux.Handle(ctx, client)
 }
+
+func handleDebug(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
+	return rpm.HandleDebug(getSpecWorker)(ctx, client)
+}
+
+func getSpecWorker(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+	st := getWorkerImage(resolver, opts...)
+	if spec.HasGomods() {
+		deps := spec.GetBuildDeps(targetKey)
+		hasGolang := func(s string) bool {
+			return s == "golang" || s == "msft-golang"
+		}
+
+		if !slices.ContainsFunc(deps, hasGolang) {
+			return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps")
+		}
+		st = st.With(installBuildDeps(spec, targetKey, opts...))
+	}
+	return st, nil
+}

frontend/rpm/handle_buildroot.go

@@ -11,15 +11,22 @@ import (
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-func HandleBuildroot() gwclient.BuildFunc {
+type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error)
+
+func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc {
 	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
 		return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
 			sOpt, err := frontend.SourceOptFromClient(ctx, client)
 			if err != nil {
 				return nil, nil, err
 			}
 
-			st, err := SpecToBuildrootLLB(spec, sOpt, targetKey)
+			worker, err := wf(sOpt.Resolver, spec, targetKey)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			st, err := SpecToBuildrootLLB(worker, spec, sOpt, targetKey)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -47,13 +54,13 @@ func HandleBuildroot() gwclient.BuildFunc {
 }
 
 // SpecToBuildrootLLB converts a dalec.Spec to an rpm buildroot
-func SpecToBuildrootLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+func SpecToBuildrootLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	if err := ValidateSpec(spec); err != nil {
 		return llb.Scratch(), fmt.Errorf("invalid spec: %w", err)
 	}
 	opts = append(opts, dalec.ProgressGroup("Create RPM buildroot"))
 
-	sources, err := Dalec2SourcesLLB(spec, sOpt, opts...)
+	sources, err := Dalec2SourcesLLB(worker, spec, sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}

frontend/rpm/handle_sources.go

@@ -10,6 +10,7 @@ import (
 	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 // TarImageRef is the image used to create tarballs of sources
@@ -41,15 +42,20 @@ func tar(src llb.State, dest string, opts ...llb.ConstraintsOpt) llb.State {
 	return worker.AddMount(outBase, llb.Scratch())
 }
 
-func HandleSources() gwclient.BuildFunc {
+func HandleSources(wf WorkerFunc) gwclient.BuildFunc {
 	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
 		return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
 			sOpt, err := frontend.SourceOptFromClient(ctx, client)
 			if err != nil {
 				return nil, nil, err
 			}
 
-			sources, err := Dalec2SourcesLLB(spec, sOpt)
+			worker, err := wf(sOpt.Resolver, spec, targetKey)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			sources, err := Dalec2SourcesLLB(worker, spec, sOpt)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -78,36 +84,19 @@ func HandleSources() gwclient.BuildFunc {
 	}
 }
 
-func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.State, error) {
+func Dalec2SourcesLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.State, error) {
 	// Sort the map keys so that the order is consistent This shouldn't be
 	// needed when MergeOp is supported, but when it is not this will improve
 	// cache hits for callers of this function.
 	sorted := dalec.SortMapKeys(spec.Sources)
 
 	out := make([]llb.State, 0, len(spec.Sources))
+
 	for _, k := range sorted {
 		src := spec.Sources[k]
 		isDir := dalec.SourceIsDir(src)
 
-		s := ""
-		switch {
-		case src.DockerImage != nil:
-			s = src.DockerImage.Ref
-		case src.Git != nil:
-			s = src.Git.URL
-		case src.HTTP != nil:
-			s = src.HTTP.URL
-		case src.Context != nil:
-			s = src.Context.Name
-		case src.Build != nil:
-			s = fmt.Sprintf("%v", src.Build.Source)
-		case src.Inline != nil:
-			s = "inline"
-		default:
-			return nil, fmt.Errorf("no non-nil source provided")
-		}
-
-		pg := dalec.ProgressGroup("Add spec source: " + k + " " + s)
+		pg := dalec.ProgressGroup("Add spec source: " + k)
 		st, err := src.AsState(k, sOpt, append(opts, pg)...)
 		if err != nil {
 			return nil, err
@@ -120,5 +109,15 @@ func Dalec2SourcesLLB(spec *dalec.Spec, sOpt dalec.SourceOpts, opts ...llb.Const
 		}
 	}
 
+	opts = append(opts, dalec.ProgressGroup("Add gomod sources"))
+	st, err := spec.GomodDeps(sOpt, worker, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "error adding gomod sources")
+	}
+
+	if st != nil {
+		out = append(out, tar(*st, gomodsName+".tar.gz", opts...))
+	}
+
 	return out, nil
 }