Skip to content

Commit

Permalink
Use native platform to fetch sources, generate buildroot, etc
Browse files Browse the repository at this point in the history 
When building for a non-native platform (typically under qemu), use the
native platform for most operations that do not need platform specific
things, such as applying patches, tarring up sources, and fetching
go module deps.

Signed-off-by: Brian Goff <[email protected]>
  • Loading branch information
@cpuguy83
cpuguy83 committed Aug 29, 2024
1 parent a05e530 commit 2f39409
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 28 deletions.
2 changes: 1 addition & 1 deletion frontend/azlinux/handle_container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc {

pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name)

rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform))
rpmDir, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg)
if err != nil {
return nil, nil, fmt.Errorf("error creating rpm: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion frontend/azlinux/handle_depsonly.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
return nil, nil, err
}

st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg)
st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg, dalec.WithPlatform(platform))
if err != nil {
return nil, nil, err
}
Expand Down
66 changes: 48 additions & 18 deletions frontend/azlinux/handle_rpm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

func handleRPM(w worker) gwclient.BuildFunc {
Expand All @@ -26,7 +27,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
return nil, nil, err
}

st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform))
st, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg)
if err != nil {
return nil, nil, err
}
Expand All @@ -53,7 +54,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
}

// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc {
func installBuildDepsPackage(target string, packageName string, w worker, sOpt dalec.SourceOpts, deps map[string]dalec.PackageConstraints, platform *ocispecs.Platform, installOpts ...installOpt) installFunc {
// depsOnly is a simple dalec spec that only includes build dependencies and their constraints
depsOnly := dalec.Spec{
Name: fmt.Sprintf("%s-build-dependencies", packageName),
Expand All @@ -66,11 +67,11 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m
},
}

return func(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (llb.RunOption, error) {
return func(ctx context.Context, Opt dalec.SourceOpts) (llb.RunOption, error) {
pg := dalec.ProgressGroup("Building container for build dependencies")

// create an RPM with just the build dependencies, using our same base worker
rpmDir, err := specToRpmLLB(ctx, w, client, &depsOnly, sOpt, target, pg)
rpmDir, err := createRPM(ctx, w, sOpt, &depsOnly, target, platform, pg)
if err != nil {
return nil, err
}
Expand All @@ -91,20 +92,15 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m
}
}

func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
func installBuildDeps(ctx context.Context, w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
deps := spec.GetBuildDeps(targetKey)
if len(deps) == 0 {
return func(in llb.State) llb.State { return in }, nil
}

sOpt, err := frontend.SourceOptFromClient(ctx, client)
if err != nil {
return nil, err
}

opts = append(opts, dalec.ProgressGroup("Install build deps"))

installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt)
installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, sOpt, deps, platform, installWithConstraints(opts))(ctx, sOpt)
if err != nil {
return nil, err
}
Expand All @@ -114,24 +110,58 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe
}, nil
}

func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
base, err := w.Base(sOpt, opts...)
func createBuildroot(ctx context.Context, w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
opts = append(opts, dalec.ProgressGroup("Prepare rpm build root"))

// Always generate the build root usign the native platform
// There is nothing it does that should require the requested target platform
native, err := w.Base(sOpt, opts...)
if err != nil {
return llb.Scratch(), err
}

installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...)
if spec.HasGomods() {
// Since the spec has go mods in it, we need to make sure we have go
// installed in the image.
nativeInstallDeps, err := installBuildDeps(ctx, w, sOpt, spec, targetKey, nil, opts...)
if err != nil {
return llb.Scratch(), err
}

native = native.With(nativeInstallDeps)
}

return rpm.SpecToBuildrootLLB(native, spec, sOpt, targetKey, opts...)
}

func createRPM(ctx context.Context, w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
br, err := createBuildroot(ctx, w, sOpt, spec, targetKey, opts...)
if err != nil {
return llb.Scratch(), errors.Wrap(err, "error creating rpm build root")
}

specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")

// Build the RPM with the target platform
opts = append(opts, dalec.WithPlatform(platform))
base, err := w.Base(sOpt, opts...)
if err != nil {
return llb.Scratch(), err
}
base = base.With(installOpt)

br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...)
installDeps, err := installBuildDeps(ctx, w, sOpt, spec, targetKey, platform, opts...)
if err != nil {
return llb.Scratch(), err
}
specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")
st := rpm.Build(br, base, specPath, opts...)

base = base.With(installDeps)
return rpm.Build(br, base, specPath, opts...), nil
}

func buildOutputRPM(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
st, err := createRPM(ctx, w, sOpt, spec, targetKey, platform, opts...)
if err != nil {
return llb.Scratch(), err
}
return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt)
}
10 changes: 5 additions & 5 deletions frontend/azlinux/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ const (
tdnfCacheDir = "/var/cache/tdnf"
)

type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.RunOption, error)
type installFunc func(context.Context, dalec.SourceOpts) (llb.RunOption, error)

type worker interface {
Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error)
Expand Down Expand Up @@ -62,12 +62,12 @@ func handleDebug(w worker) gwclient.BuildFunc {
if err != nil {
return nil, err
}
return rpm.HandleDebug(getSpecWorker(ctx, w, client, sOpt))(ctx, client)
return rpm.HandleDebug(getSpecWorker(ctx, w, sOpt))(ctx, client)
}
}

func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt dalec.SourceOpts) rpm.WorkerFunc {
return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
func getSpecWorker(ctx context.Context, w worker, sOpt dalec.SourceOpts) rpm.WorkerFunc {
return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) {
st, err := w.Base(sOpt, opts...)
if err != nil {
return llb.Scratch(), err
Expand All @@ -83,7 +83,7 @@ func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt d
return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps")
}

installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...)
installOpt, err := installBuildDeps(ctx, w, sOpt, spec, targetKey, platform, opts...)
if err != nil {
return llb.Scratch(), err
}
Expand Down
6 changes: 4 additions & 2 deletions frontend/rpm/handle_buildroot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
)

type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error)
type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error)

func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc {
return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
Expand All @@ -21,7 +21,9 @@ func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc {
return nil, nil, err
}

worker, err := wf(sOpt.Resolver, spec, targetKey)
// Note, we are not passing platform down here because everything should
// be able to work regardless of platform, so prefer the native platform.
worker, err := wf(sOpt.Resolver, spec, targetKey, platform)
if err != nil {
return nil, nil, err
}
Expand Down
4 changes: 3 additions & 1 deletion frontend/rpm/handle_sources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ func HandleSources(wf WorkerFunc) gwclient.BuildFunc {
return nil, nil, err
}

worker, err := wf(sOpt.Resolver, spec, targetKey)
worker, err := wf(sOpt.Resolver, spec, targetKey, platform)
if err != nil {
return nil, nil, err
}

// Note, we are not passing platform down here because everything should
// be able to work regardless of platform, so prefer the native platform.
sources, err := Dalec2SourcesLLB(worker, spec, sOpt)
if err != nil {
return nil, nil, err
Expand Down

0 comments on commit 2f39409

Please sign in to comment.