Custom Repo Fixes

docs/spec.schema.json

@@ -836,6 +836,10 @@
 				"digest": {
 					"type": "string",
 					"description": "Digest is the digest of the file to download.\nThis is used to verify the integrity of the file.\nForm: \u003calgorithm\u003e:\u003cdigest\u003e"
+				},
+				"permissions": {
+					"type": "integer",
+					"description": "Permissions is the octal file permissions to set on the file."
 				}
 			},
 			"additionalProperties": false,

frontend/azlinux/handle_container.go

@@ -56,7 +56,7 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 	}
 
 	installTimeRepos := spec.GetInstallRepos(targetKey)
-	importRepos, err := repoMountInstallOpts(w, builderImg, installTimeRepos, sOpt, opts...)
+	importRepos, err := repoMountInstallOpts(installTimeRepos, sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}

frontend/azlinux/handle_rpm.go

@@ -97,21 +97,14 @@ func runTests(ctx context.Context, client gwclient.Client, w worker, spec *dalec
 	return ref, errors.Wrap(err, "TESTS FAILED")
 }
 
-var azLinuxRepoConfig = dalec.RepoPlatformConfig{
+var azlinuxRepoPlatform = dalec.RepoPlatformConfig{
 	ConfigRoot: "/etc/yum.repos.d",
 	GPGKeyRoot: "/etc/pki/rpm-gpg",
+	ConfigExt:  ".repo",
 }
 
-func repoMountInstallOpts(w worker, base llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) {
-	worker := base.Run(
-		w.Install(
-			[]string{"gnupg2"},
-			installWithConstraints(opts),
-		),
-		dalec.WithConstraints(opts...),
-	).Root()
-
-	withRepos, err := dalec.WithRepoConfigs(repos, &azLinuxRepoConfig, sOpt, opts...)
+func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) {
+	withRepos, err := dalec.WithRepoConfigs(repos, &azlinuxRepoPlatform, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +114,7 @@ func repoMountInstallOpts(w worker, base llb.State, repos []dalec.PackageReposit
 		return nil, err
 	}
 
-	keyMounts, keyPaths, err := dalec.GetRepoKeys(worker, repos, &azLinuxRepoConfig, sOpt, opts...)
+	keyMounts, keyPaths, err := dalec.GetRepoKeys(repos, &azlinuxRepoPlatform, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -137,7 +130,7 @@ func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey s
 	}
 
 	testRepos := spec.GetTestRepos(targetKey)
-	importRepos, err := repoMountInstallOpts(w, base, testRepos, sOpt, opts...)
+	importRepos, err := repoMountInstallOpts(testRepos, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -210,12 +203,7 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe
 		return nil, err
 	}
 
-	base, err := w.Base(sOpt, opts...)
-	if err != nil {
-		return nil, err
-	}
-
-	importRepos, err := repoMountInstallOpts(w, base, repos, sOpt, opts...)
+	importRepos, err := repoMountInstallOpts(repos, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}

frontend/jammy/handle_container.go

@@ -90,7 +90,7 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 			AddMount(workPath, in)
 	}
 
-	customRepoOpts, err := customRepoMounts(worker, spec.GetInstallRepos(targetKey), sOpt, opts...)
+	customRepoOpts, err := customRepoMounts(spec.GetInstallRepos(targetKey), sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}
@@ -131,13 +131,13 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 		With(installSymlinks), nil
 }
 
-func installTestDeps(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
+func installTestDeps(spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
 	deps := spec.GetTestDeps(targetKey)
 	if len(deps) == 0 {
 		return func(s llb.State) llb.State { return s }, nil
 	}
 
-	extraRepoOpts, err := customRepoMounts(worker, spec.GetTestRepos(targetKey), sOpt, opts...)
+	extraRepoOpts, err := customRepoMounts(spec.GetTestRepos(targetKey), sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}

frontend/jammy/handle_deb.go

@@ -111,7 +111,7 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp
 		return nil, err
 	}
 
-	withTestDeps, err := installTestDeps(worker, spec, sOpt, targetKey, opts...)
+	withTestDeps, err := installTestDeps(spec, sOpt, targetKey, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -120,14 +120,14 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp
 	return ref, err
 }
 
-var jammyRepoPlatformCfg = dalec.RepoPlatformConfig{
+var jammyRepoPlatform = dalec.RepoPlatformConfig{
 	ConfigRoot: "/etc/apt/sources.list.d",
 	GPGKeyRoot: "/usr/share/keyrings",
+	ConfigExt:  ".list",
 }
 
-func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
-	worker = worker.Run(installPackages([]string{"gnupg2"}, opts...), dalec.WithConstraints(opts...)).Root() // make sure we have gpg installed
-	withRepos, err := dalec.WithRepoConfigs(repos, &jammyRepoPlatformCfg, sOpt, opts...)
+func customRepoMounts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	withRepos, err := dalec.WithRepoConfigs(repos, &jammyRepoPlatform, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -137,7 +137,7 @@ func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, s
 		return nil, err
 	}
 
-	keyMounts, _, err := dalec.GetRepoKeys(worker, repos, &jammyRepoPlatformCfg, sOpt, opts...)
+	keyMounts, _, err := dalec.GetRepoKeys(repos, &jammyRepoPlatform, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -319,7 +319,7 @@ func buildDepends(worker llb.State, sOpt dalec.SourceOpts, spec *dalec.Spec, tar
 		return nil, errors.Wrap(err, "error creating intermediate package for installing build dependencies")
 	}
 
-	customRepoOpts, err := customRepoMounts(worker, spec.GetBuildRepos(targetKey), sOpt, opts...)
+	customRepoOpts, err := customRepoMounts(spec.GetBuildRepos(targetKey), sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}

frontend/windows/handle_zip.go

@@ -117,7 +117,7 @@ func installBuildDeps(sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string)
 				return in, errors.Wrap(err, "error creating intermediate package for installing build dependencies")
 			}
 
-			customRepoOpts, err := customRepoMounts(in, spec.GetBuildRepos(targetKey), sOpt, opts...)
+			customRepoOpts, err := customRepoMounts(spec.GetBuildRepos(targetKey), sOpt, opts...)
 			if err != nil {
 				return in, err
 			}
@@ -315,13 +315,7 @@ var jammyRepoPlatformCfg = dalec.RepoPlatformConfig{
 	GPGKeyRoot: "/usr/share/keyrings",
 }
 
-func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
-	worker = worker.Run(
-		dalec.ShArgs("apt update && apt install -y gnupg2"),
-		dalec.WithMountedAptCache(aptCachePrefix),
-		dalec.WithConstraints(opts...),
-	).Root() // make sure we have gpg installed
-
+func customRepoMounts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
 	withRepos, err := dalec.WithRepoConfigs(repos, &jammyRepoPlatformCfg, sOpt, opts...)
 	if err != nil {
 		return nil, err
@@ -332,7 +326,7 @@ func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, s
 		return nil, err
 	}
 
-	keyMounts, _, err := dalec.GetRepoKeys(worker, repos, &jammyRepoPlatformCfg, sOpt, opts...)
+	keyMounts, _, err := dalec.GetRepoKeys(repos, &jammyRepoPlatformCfg, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}

helpers.go

@@ -468,6 +468,7 @@ func (f gitOptionFunc) SetGitOption(gi *llb.GitInfo) {
 type RepoPlatformConfig struct {
 	ConfigRoot string
 	GPGKeyRoot string
+	ConfigExt  string
 }
 
 // Returns a run option which mounts the data dirs for all specified repos
@@ -508,8 +509,13 @@ func repoConfigAsMount(config PackageRepositoryConfig, platformCfg *RepoPlatform
 			return nil, err
 		}
 
+		var normalized string = name
+		if filepath.Ext(normalized) != platformCfg.ConfigExt {
+			normalized += platformCfg.ConfigExt
+		}
+
 		repoConfigs = append(repoConfigs,
-			llb.AddMount(filepath.Join(platformCfg.ConfigRoot, name), repoConfigSt, llb.SourcePath(name)))
+			llb.AddMount(filepath.Join(platformCfg.ConfigRoot, normalized), repoConfigSt, llb.SourcePath(name)))
 	}
 
 	return repoConfigs, nil
@@ -530,32 +536,19 @@ func WithRepoConfigs(repos []PackageRepositoryConfig, cfg *RepoPlatformConfig, s
 	return WithRunOptions(configStates...), nil
 }
 
-func GetRepoKeys(worker llb.State, configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
+func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
 	keys := []llb.RunOption{}
 	names := []string{}
 	for _, config := range configs {
 		for name, repoKey := range config.Keys {
-			// each of these sources represent a gpg key file for a particular repo
-			gpgKey, err := repoKey.AsState(name, sOpt, append(opts, ProgressGroup("Importing repo key: "+name))...)
+			gpgKey, err := repoKey.AsState(name, sOpt, append(opts, ProgressGroup("Fetching repo key: "+name))...)
 			if err != nil {
 				return nil, nil, err
 			}
 
 			mountPath := filepath.Join(cfg.GPGKeyRoot, name)
 
-			opt := runOptionFunc(func(ei *llb.ExecInfo) {
-				inPath := filepath.Join("/tmp/in", name)
-				outPath := filepath.Join("/tmp/out", name)
-				keySt := worker.Run(
-					// dearmor key if necessary
-					ShArgs(fmt.Sprintf("gpg --dearmor --output %q < %q", outPath, inPath)),
-					llb.AddMount(inPath, gpgKey, llb.SourcePath(name))).
-					AddMount("/tmp/out/", llb.Scratch())
-
-				llb.AddMount(mountPath, keySt, llb.SourcePath(name)).SetRunOption(ei)
-			})
-
-			keys = append(keys, opt)
+			keys = append(keys, llb.AddMount(mountPath, gpgKey, llb.SourcePath(name)))
 			names = append(names, name)
 		}
 	}

load.go

@@ -453,6 +453,42 @@ func (s *Spec) FillDefaults() {
 			s.Patches[k][i].Strip = &strip
 		}
 	}
+
+	if s.Dependencies != nil {
+		for i := range len(s.Dependencies.ExtraRepos) {
+			fillExtraRepoDefaults(&s.Dependencies.ExtraRepos[i])
+		}
+	}
+}
+
+func fillExtraRepoDefaults(extraRepo *PackageRepositoryConfig) {
+	if len(extraRepo.Envs) == 0 {
+		// default to all stages for the extra repo if unspecified
+		extraRepo.Envs = []string{"build", "install", "test"}
+	}
+
+	for configName := range extraRepo.Config {
+		configSource := extraRepo.Config[configName]
+		fillDefaults(&configSource)
+		extraRepo.Config[configName] = configSource
+	}
+
+	for keyName := range extraRepo.Keys {
+		keySource := extraRepo.Keys[keyName]
+		fillDefaults(&keySource)
+
+		// Default to 0644 permissions for gpg keys. This is because apt will will only import
+		// keys with a particular permission set.
+		if keySource.HTTP != nil {
+			keySource.HTTP.Permissions = 0644
+		}
+
+		extraRepo.Keys[keyName] = keySource
+	}
+
+	for _, mount := range extraRepo.Data {
+		fillDefaults(&mount.Spec)
+	}
 }
 
 func (s Spec) Validate() error {

load_test.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/moby/buildkit/frontend/dockerui"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/assert/cmp"
 )
@@ -604,6 +605,44 @@ func TestSpec_SubstituteBuildArgs(t *testing.T) {
 	assert.Check(t, cmp.Equal(spec.Targets["t2"].PackageConfig.Signer.Args["REGULAR"], plainOleValue))
 }
 
+func TestCustomRepoFillDefaults(t *testing.T) {
+	// In this case, the context source for the repo config and provided public key are not set,
+	// so they should be set to the default context per source default-filling conventions.
+
+	// Also, the env field should be set to all build stages, "build", "install", and "test", as it is
+	// unspecified
+	dt := []byte(`
+dependencies:
+  extra_repos: 
+    - config:
+        custom.repo:
+          context: {}
+      keys:
+        public.gpg:
+          context: {}
+          path: "public.gpg"
+	`)
+
+	spec, err := LoadSpec(dt)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = spec.SubstituteArgs(map[string]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	extraRepo := spec.Dependencies.ExtraRepos[0]
+	assert.Equal(t, extraRepo.Config["custom.repo"].Context.Name,
+		dockerui.DefaultLocalNameContext)
+
+	assert.Equal(t, extraRepo.Keys["public.gpg"].Context.Name,
+		dockerui.DefaultLocalNameContext)
+
+	assert.DeepEqual(t, extraRepo.Envs, []string{"build", "install", "test"})
+}
+
 func TestBuildArgSubst(t *testing.T) {
 	t.Run("value provided", func(t *testing.T) {
 		dt := []byte(`

source.go

@@ -164,6 +164,11 @@ func (src *SourceHTTP) AsState(name string, opts ...llb.ConstraintsOpt) (llb.Sta
 	if src.Digest != "" {
 		httpOpts = append(httpOpts, llb.Checksum(src.Digest))
 	}
+
+	if src.Permissions != 0 {
+		httpOpts = append(httpOpts, llb.Chmod(src.Permissions))
+	}
+
 	st := llb.HTTP(src.URL, httpOpts...)
 	return st, nil
 }

spec.go

@@ -197,6 +197,8 @@ type SourceHTTP struct {
 	// This is used to verify the integrity of the file.
 	// Form: <algorithm>:<digest>
 	Digest digest.Digest `yaml:"digest,omitempty" json:"digest,omitempty"`
+	// Permissions is the octal file permissions to set on the file.
+	Permissions fs.FileMode `yaml:"permissions,omitempty" json:"permissions,omitempty"`
 }
 
 // SourceContext is used to generate a source from a build context. The path to

test/azlinux_test.go

@@ -28,21 +28,23 @@ var azlinuxConstraints = constraintsSymbols{
 	LessThanOrEqual:    "<=",
 }
 
-var azlinuxTestRepoConfig = map[string]dalec.Source{
-	"local.repo": {
-		Inline: &dalec.SourceInline{
-			File: &dalec.SourceInlineFile{
-				Contents: `[Local]
+var azlinuxTestRepoConfig = func(keyPath string) map[string]dalec.Source {
+	return map[string]dalec.Source{
+		"local.repo": {
+			Inline: &dalec.SourceInline{
+				File: &dalec.SourceInlineFile{
+					Contents: fmt.Sprintf(`[Local]
 name=Local Repository
 baseurl=file:///opt/repo
 repo_gpgcheck=1
 priority=0
 enabled=1
-gpgkey=file:///etc/pki/rpm-gpg/public.key
-	`,
+gpgkey=file:///etc/pki/rpm-gpg/%s
+	`, keyPath),
+				},
 			},
 		},
-	},
+	}
 }
 
 func TestMariner2(t *testing.T) {
@@ -169,7 +171,7 @@ type workerConfig struct {
 	// ContextName is the name of the worker context that the build target will use
 	// to see if a custom worker is proivded in a context
 	ContextName    string
-	TestRepoConfig map[string]dalec.Source
+	TestRepoConfig func(string) map[string]dalec.Source
 	Constraints    constraintsSymbols
 	Platform       *ocispecs.Platform
 }