From 66212c0128db2665235204be884b570b76d03ac5 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 14 Oct 2023 05:39:35 +0200 Subject: [PATCH] bake: display read definition files in build output Signed-off-by: CrazyMax --- bake/bake.go | 108 ++++++++++++++++++++++-- bake/bake_test.go | 2 +- bake/remote.go | 33 ++++++-- commands/bake.go | 5 +- tests/bake.go | 38 +++++++++ util/cobrautil/completion/completion.go | 2 +- 6 files changed, 171 insertions(+), 17 deletions(-) diff --git a/bake/bake.go b/bake/bake.go index dbe839b2f83..fa8c8dac459 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "time" composecli "github.com/compose-spec/compose-go/cli" "github.com/docker/buildx/bake/hclparser" @@ -18,8 +19,10 @@ import ( controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/platformutil" + "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/config" hcl "github.com/hashicorp/hcl/v2" + "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session/auth/authprovider" "github.com/pkg/errors" @@ -54,7 +57,7 @@ func defaultFilenames() []string { return names } -func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { +func ReadLocalFiles(names []string, stdin io.Reader, l progress.SubLogger) ([]File, error) { isDefault := false if len(names) == 0 { isDefault = true @@ -62,20 +65,26 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { } out := make([]File, 0, len(names)) + setStatus := func(st *client.VertexStatus) { + if l != nil { + l.SetStatus(st) + } + } + for _, n := range names { var dt []byte var err error if n == "-" { - dt, err = io.ReadAll(stdin) + dt, err = readWithStatus(stdin, setStatus) if err != nil { return nil, err } } else { - dt, err = os.ReadFile(n) + dt, err = readFileWithStatus(n, isDefault, setStatus) + if dt == nil && err == nil { + continue + } if err != nil { - if isDefault && errors.Is(err, os.ErrNotExist) { - continue - } return nil, err } } @@ -84,6 +93,93 @@ func ReadLocalFiles(names []string, stdin io.Reader) ([]File, error) { return out, nil } +func readFileWithStatus(fname string, isDefault bool, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading " + fname, + } + + defer func() { + now := time.Now() + st.Completed = &now + if dt != nil || err != nil { + setStatus(st) + } + }() + + now := time.Now() + st.Started = &now + + f, err := os.Open(fname) + if err != nil { + if isDefault && errors.Is(err, os.ErrNotExist) { + return nil, nil + } + setStatus(st) + return nil, err + } + defer f.Close() + setStatus(st) + + info, err := f.Stat() + if err != nil { + return nil, err + } + st.Total = info.Size() + setStatus(st) + + buf := make([]byte, 1024) + current := int64(0) + for { + n, err := f.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + current += int64(n) + st.Current = current + setStatus(st) + } + + return dt, nil +} + +func readWithStatus(r io.Reader, setStatus func(st *client.VertexStatus)) (dt []byte, err error) { + st := &client.VertexStatus{ + ID: "reading from stdin", + } + + defer func() { + now := time.Now() + st.Completed = &now + setStatus(st) + }() + + now := time.Now() + st.Started = &now + setStatus(st) + + buf := make([]byte, 1024) + current := int64(0) + for { + n, err := r.Read(buf) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + dt = append(dt, buf[:n]...) + current += int64(n) + st.Current = current + setStatus(st) + } + + return dt, nil +} + func ListTargets(files []File) ([]string, error) { c, err := ParseFiles(files, nil) if err != nil { diff --git a/bake/bake_test.go b/bake/bake_test.go index a54e66c9230..1fbd631d0eb 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -1408,7 +1408,7 @@ func TestReadLocalFilesDefault(t *testing.T) { for _, tf := range tt.filenames { require.NoError(t, os.WriteFile(tf, []byte(tf), 0644)) } - files, err := ReadLocalFiles(nil, nil) + files, err := ReadLocalFiles(nil, nil, nil) require.NoError(t, err) if len(files) == 0 { require.Equal(t, len(tt.expected), len(files)) diff --git a/bake/remote.go b/bake/remote.go index 3dd1e0d1679..855cce67bcc 100644 --- a/bake/remote.go +++ b/bake/remote.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "context" + "time" "github.com/docker/buildx/builder" controllerapi "github.com/docker/buildx/controller/pb" @@ -76,11 +77,14 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name return nil, err } - if filename != "" { - files, err = filesFromURLRef(ctx, c, ref, inp, filename, names) - } else { - files, err = filesFromRef(ctx, ref, names) - } + progress.Wrap("[internal] load remote bake definitions", pw.Write, func(sub progress.SubLogger) error { + if filename != "" { + files, err = filesFromURLRef(ctx, c, ref, inp, filename, names, sub) + } else { + files, err = filesFromRef(ctx, ref, names, sub) + } + return nil + }) return nil, err }, ch) @@ -110,7 +114,7 @@ func isArchive(header []byte) bool { return err == nil } -func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) { +func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string, l progress.SubLogger) ([]File, error) { stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename}) if err != nil { return nil, err @@ -148,13 +152,20 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere return nil, err } - return filesFromRef(ctx, ref, names) + return filesFromRef(ctx, ref, names, l) } inp.State = nil name := inp.URL inp.URL = "" + now := time.Now() + l.SetStatus(&client.VertexStatus{ + ID: "reading " + name, + Started: &now, + Completed: &now, + }) + if len(dt) > stat.Size() { if stat.Size() > 1024*512 { return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size") @@ -171,7 +182,7 @@ func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Refere return []File{{Name: name, Data: dt}}, nil } -func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) { +func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string, l progress.SubLogger) ([]File, error) { // TODO: auto-remove parent dir in needed var files []File @@ -189,6 +200,12 @@ func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ( } return nil, err } + now := time.Now() + l.SetStatus(&client.VertexStatus{ + ID: "reading " + name, + Started: &now, + Completed: &now, + }) dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name}) if err != nil { return nil, err diff --git a/commands/bake.go b/commands/bake.go index 4843233f14b..48542482b35 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -150,7 +150,10 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions, cFlags com if url != "" { files, inp, err = bake.ReadRemoteFiles(ctx, nodes, url, in.files, printer) } else { - files, err = bake.ReadLocalFiles(in.files, dockerCli.In()) + progress.Wrap("[internal] load local bake definitions", printer.Write, func(sub progress.SubLogger) error { + files, err = bake.ReadLocalFiles(in.files, dockerCli.In(), sub) + return nil + }) } if err != nil { return err diff --git a/tests/bake.go b/tests/bake.go index b76a8bb8a2f..b00f064ee6d 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -19,6 +19,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) { var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakeLocal, + testBakeLocalMulti, testBakeRemote, testBakeRemoteCmdContext, testBakeRemoteCmdContextOverride, @@ -46,6 +47,41 @@ target "default" { out, err := bakeCmd(sb, withDir(dir), withArgs("--set", "*.output=type=local,dest="+dirDest)) require.NoError(t, err, out) + require.Contains(t, out, `#1 [internal] load local bake definitions`) + require.Contains(t, out, `#1 [internal] reading docker-bake.hcl`) + + require.FileExists(t, filepath.Join(dirDest, "foo")) +} + +func testBakeLocalMulti(t *testing.T, sb integration.Sandbox) { + dockerfile := []byte(` +FROM scratch +COPY foo /foo + `) + bakefile := []byte(` +target "default" { +} +`) + composefile := []byte(` +services: + app: + build: {} +`) + + dir := tmpdir( + t, + fstest.CreateFile("docker-bake.hcl", bakefile, 0600), + fstest.CreateFile("compoose.yaml", composefile, 0600), + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("foo", []byte("foo"), 0600), + ) + dirDest := t.TempDir() + + out, err := bakeCmd(sb, withDir(dir), withArgs("--set", "*.output=type=local,dest="+dirDest)) + require.NoError(t, err, out) + require.Contains(t, out, `#1 [internal] load local bake definitions`) + require.Contains(t, out, `#1 [internal] reading compose.yaml`) + require.Contains(t, out, `#1 [internal] reading docker-bake.hcl`) require.FileExists(t, filepath.Join(dirDest, "foo")) } @@ -76,6 +112,8 @@ EOT out, err := bakeCmd(sb, withDir(dir), withArgs(addr, "--set", "*.output=type=local,dest="+dirDest)) require.NoError(t, err, out) + require.Contains(t, out, `#1 [internal] load remote bake definitions`) + require.Contains(t, out, `#1 [internal] reading docker-bake.hcl`) require.FileExists(t, filepath.Join(dirDest, "foo")) } diff --git a/util/cobrautil/completion/completion.go b/util/cobrautil/completion/completion.go index 60754537802..095cac52109 100644 --- a/util/cobrautil/completion/completion.go +++ b/util/cobrautil/completion/completion.go @@ -19,7 +19,7 @@ func Disable(cmd *cobra.Command, args []string, toComplete string) ([]string, co func BakeTargets(files []string) ValidArgsFn { return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - f, err := bake.ReadLocalFiles(files, nil) + f, err := bake.ReadLocalFiles(files, nil, nil) if err != nil { return nil, cobra.ShellCompDirectiveError }