build: support local state group

Signed-off-by: CrazyMax <[email protected]>

bake/bake.go

@@ -33,6 +33,11 @@ var (
 	targetNamePattern    = regexp.MustCompile(`^` + validTargetNameChars + `$`)
 )
 
+type Definition struct {
+	Group  map[string]*Group  `json:"group,omitempty"`
+	Target map[string]*Target `json:"target"`
+}
+
 type File struct {
 	Name string
 	Data []byte

build/build.go

@@ -26,6 +26,7 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/buildx/util/desktop"
 	"github.com/docker/buildx/util/dockerutil"
 	"github.com/docker/buildx/util/imagetools"
@@ -72,6 +73,7 @@ const (
 type Build struct {
 	ctx              context.Context
 	resultHandleFunc func(driverIndex int, rCtx *ResultHandle)
+	localstateGroup  *localstate.StateGroup
 }
 
 // Option provides a variadic option for configuring the build.
@@ -83,6 +85,12 @@ func WithResultHandleFunc(resultHandleFunc func(driverIndex int, rCtx *ResultHan
 	}
 }
 
+func WithLocalStateGroup(localstateGroup *localstate.StateGroup) Option {
+	return func(b *Build) {
+		b.localstateGroup = localstateGroup
+	}
+}
+
 // New initializes a new build client
 func New(ctx context.Context, opts ...Option) *Build {
 	b := &Build{
@@ -765,7 +773,7 @@ func (b *Build) Run(nodes []builder.Node, opt map[string]Options, docker *docker
 			if err != nil {
 				return nil, err
 			}
-			if err := saveLocalState(so, k, opt, node, configDir); err != nil {
+			if err := saveLocalStateRef(so, k, opt, b.localstateGroup, node, configDir); err != nil {
 				return nil, err
 			}
 			for k, v := range gitattrs {
@@ -798,6 +806,10 @@ func (b *Build) Run(nodes []builder.Node, opt map[string]Options, docker *docker
 		}
 	}
 
+	if err := saveLocalStateGroup(b.localstateGroup, configDir); err != nil {
+		return nil, err
+	}
+
 	// validate that all links between targets use same drivers
 	for name := range opt {
 		dps := m[name]

build/localstate.go

@@ -1,14 +1,15 @@
 package build
 
 import (
+	"fmt"
 	"path/filepath"
 
 	"github.com/docker/buildx/builder"
 	"github.com/docker/buildx/localstate"
 	"github.com/moby/buildkit/client"
 )
 
-func saveLocalState(so *client.SolveOpt, target string, opts Options, node builder.Node, configDir string) error {
+func saveLocalStateRef(so *client.SolveOpt, target string, opts Options, lsg *localstate.StateGroup, node builder.Node, configDir string) error {
 	var err error
 	if so.Ref == "" {
 		return nil
@@ -28,17 +29,31 @@ func saveLocalState(so *client.SolveOpt, target string, opts Options, node build
 				return err
 			}
 		}
-		ls, err := localstate.New(configDir)
-		if err != nil {
-			return err
-		}
-		if err := ls.SaveRef(node.Builder, node.Name, so.Ref, localstate.State{
+		ls := localstate.State{
 			Target:         target,
 			LocalPath:      lp,
 			DockerfilePath: dp,
-		}); err != nil {
+		}
+		l, err := localstate.New(configDir)
+		if err != nil {
 			return err
 		}
+		if lsg != nil {
+			ls.GroupID = lsg.Ref
+			lsg.Refs = append(lsg.Refs, fmt.Sprintf("%s/%s/%s", node.Builder, node.Name, so.Ref))
+		}
+		return l.SaveRef(node.Builder, node.Name, so.Ref, ls)
 	}
 	return nil
 }
+
+func saveLocalStateGroup(lsg *localstate.StateGroup, configDir string) error {
+	if lsg == nil || len(lsg.Refs) == 0 {
+		return nil
+	}
+	l, err := localstate.New(configDir)
+	if err != nil {
+		return err
+	}
+	return l.SaveGroup(lsg.Ref, *lsg)
+}

commands/bake.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/buildx/bake"
 	"github.com/docker/buildx/build"
 	"github.com/docker/buildx/builder"
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/cobrautil/completion"
 	"github.com/docker/buildx/util/confutil"
@@ -19,6 +20,7 @@ import (
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -161,6 +163,10 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 	if err != nil {
 		return err
 	}
+	def := bake.Definition{
+		Group:  grps,
+		Target: tgts,
+	}
 
 	if v := os.Getenv("SOURCE_DATE_EPOCH"); v != "" {
 		// TODO: extract env var parsing to a method easily usable by library consumers
@@ -182,13 +188,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 	}
 
 	if in.printOnly {
-		dt, err := json.MarshalIndent(struct {
-			Group  map[string]*bake.Group  `json:"group,omitempty"`
-			Target map[string]*bake.Target `json:"target"`
-		}{
-			grps,
-			tgts,
-		}, "", "  ")
+		dt, err := json.MarshalIndent(def, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -201,7 +201,21 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com
 		return nil
 	}
 
-	resp, err := build.New(ctx).Run(nodes, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer)
+	var lsg *localstate.StateGroup
+	if len(bo) > 1 {
+		dt, err := json.Marshal(def)
+		if err != nil {
+			return err
+		}
+		lsg = &localstate.StateGroup{
+			Ref:        identity.NewID(),
+			Definition: dt,
+			Targets:    targets,
+			Inputs:     overrides,
+		}
+	}
+
+	resp, err := build.New(ctx, build.WithLocalStateGroup(lsg)).Run(nodes, bo, dockerutil.NewClient(dockerCli), confutil.ConfigDir(dockerCli), printer)
 	if err != nil {
 		return wrapBuildError(err, true)
 	}

localstate/localstate.go

@@ -9,12 +9,33 @@ import (
 	"github.com/pkg/errors"
 )
 
-const refsDir = "refs"
+const (
+	refsDir  = "refs"
+	groupDir = "__group__"
+)
 
 type State struct {
-	Target         string
-	LocalPath      string
+	// Target is the name of the target
+	Target string
+	// LocalPath is the absolute path to the context
+	LocalPath string
+	// DockerfilePath is the absolute path to the Dockerfile
 	DockerfilePath string
+	// GroupID is the ID of the state group that this ref belongs to
+	GroupID string `json:",omitempty"`
+}
+
+type StateGroup struct {
+	// Ref is the unique ID of the group ref
+	Ref string `json:"-"`
+	// Definition is the raw representation of the group (bake definition)
+	Definition []byte
+	// Targets are the targets invoked
+	Targets []string `json:",omitempty"`
+	// Inputs are the inputs of the group
+	Inputs []string `json:",omitempty"`
+	// Refs are used to track all the refs that belong to the same group
+	Refs []string
 }
 
 type LocalState struct {
@@ -63,6 +84,30 @@ func (ls *LocalState) SaveRef(builderName, nodeName, id string, st State) error
 	return ioutils.AtomicWriteFile(filepath.Join(refDir, id), dt, 0600)
 }
 
+func (ls *LocalState) ReadGroup(id string) (*StateGroup, error) {
+	dt, err := os.ReadFile(filepath.Join(ls.root, refsDir, groupDir, id))
+	if err != nil {
+		return nil, err
+	}
+	var stg StateGroup
+	if err := json.Unmarshal(dt, &stg); err != nil {
+		return nil, err
+	}
+	return &stg, nil
+}
+
+func (ls *LocalState) SaveGroup(id string, stg StateGroup) error {
+	refDir := filepath.Join(ls.root, refsDir, groupDir)
+	if err := os.MkdirAll(refDir, 0700); err != nil {
+		return err
+	}
+	dt, err := json.Marshal(stg)
+	if err != nil {
+		return err
+	}
+	return ioutils.AtomicWriteFile(filepath.Join(refDir, id), dt, 0600)
+}
+
 func (ls *LocalState) RemoveBuilder(builderName string) error {
 	if builderName == "" {
 		return errors.Errorf("builder name empty")