Skip to content

Commit

Permalink
builder: move kube config handling to k8s driver package
Browse files Browse the repository at this point in the history 
Signed-off-by: CrazyMax <[email protected]>
  • Loading branch information
@crazy-max
crazy-max committed Jul 16, 2024
1 parent 4304d38 commit 691c575
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 89 deletions.
4 changes: 2 additions & 2 deletions builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ func (b *Builder) Factory(ctx context.Context, dialMeta map[string][]string) (_
if _, err = dockerapi.Ping(ctx); err != nil {
return
}
b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, ep, dockerapi, false, dialMeta)
b.driverFactory.Factory, err = driver.GetDefaultFactory(ctx, b.opts.dockerCli, ep, dockerapi, false, dialMeta)
if err != nil {
return
}
Expand Down Expand Up @@ -394,7 +394,7 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre
} else if opts.Endpoint == "" && buildkitHost != "" {
driverName = "remote"
} else {
f, err := driver.GetDefaultFactory(ctx, opts.Endpoint, dockerCli.Client(), true, nil)
f, err := driver.GetDefaultFactory(ctx, dockerCli, opts.Endpoint, dockerCli.Client(), true, nil)
if err != nil {
return nil, err
}
Expand Down
54 changes: 11 additions & 43 deletions builder/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/containerd/platforms"
"github.com/docker/buildx/driver"
ctxkube "github.com/docker/buildx/driver/kubernetes/context"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/dockerutil"
Expand All @@ -18,7 +17,6 @@ import (
"github.com/moby/buildkit/util/grpcerrors"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
)
Expand Down Expand Up @@ -119,47 +117,17 @@ func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []N
return nil
}

contextStore := b.opts.dockerCli.ContextStore()

var kcc driver.KubeClientConfig
kcc, err = ctxkube.ConfigFromEndpoint(n.Endpoint, contextStore)
if err != nil {
// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
// try again with name="default".
// FIXME(@AkihiroSuda): n should retain real context name.
kcc, err = ctxkube.ConfigFromEndpoint("default", contextStore)
if err != nil {
logrus.Error(err)
}
}

tryToUseKubeConfigInCluster := false
if kcc == nil {
tryToUseKubeConfigInCluster = true
} else {
if _, err := kcc.ClientConfig(); err != nil {
tryToUseKubeConfigInCluster = true
}
}
if tryToUseKubeConfigInCluster {
kccInCluster := driver.KubeClientConfigInCluster{}
if _, err := kccInCluster.ClientConfig(); err == nil {
logrus.Debug("using kube config in cluster")
kcc = kccInCluster
}
}
d, err := driver.GetDriver(ctx, factory, driver.InitConfig{
Name: driver.BuilderName(n.Name),
EndpointAddr: n.Endpoint,
DockerAPI: dockerapi,
KubeClientConfig: kcc,
BuildkitdFlags: n.BuildkitdFlags,
Files: n.Files,
DriverOpts: n.DriverOpts,
Auth: imageopt.Auth,
Platforms: n.Platforms,
ContextPathHash: b.opts.contextPathHash,
DialMeta: lno.dialMeta,
d, err := driver.GetDriver(ctx, factory, b.opts.dockerCli, driver.InitConfig{
Name: driver.BuilderName(n.Name),
EndpointAddr: n.Endpoint,
DockerAPI: dockerapi,
BuildkitdFlags: n.BuildkitdFlags,
Files: n.Files,
DriverOpts: n.DriverOpts,
Auth: imageopt.Auth,
Platforms: n.Platforms,
ContextPathHash: b.opts.contextPathHash,
DialMeta: lno.dialMeta,
})
if err != nil {
node.Err = err
Expand Down
65 changes: 59 additions & 6 deletions driver/kubernetes/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,23 @@ package kubernetes

import (
"context"
"os"
"strconv"
"strings"
"time"

corev1 "k8s.io/api/core/v1"

"github.com/docker/buildx/driver"
"github.com/docker/buildx/driver/bkimage"
ctxkube "github.com/docker/buildx/driver/kubernetes/context"
"github.com/docker/buildx/driver/kubernetes/manifest"
"github.com/docker/buildx/driver/kubernetes/podchooser"
"github.com/docker/cli/cli/command"
dockerclient "github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)

const (
Expand All @@ -23,6 +27,25 @@ const (
defaultTimeout = 120 * time.Second
)

type ClientConfig interface {
ClientConfig() (*rest.Config, error)
Namespace() (string, bool, error)
}

type ClientConfigInCluster struct{}

func (k ClientConfigInCluster) ClientConfig() (*rest.Config, error) {
return rest.InClusterConfig()
}

func (k ClientConfigInCluster) Namespace() (string, bool, error) {
namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
if err != nil {
return "", false, err
}
return strings.TrimSpace(string(namespace)), true, nil
}

func init() {
driver.Register(&factory{})
}
Expand All @@ -45,19 +68,49 @@ func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.
return prioritySupported
}

func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) {
if cfg.KubeClientConfig == nil {
func (f *factory) New(ctx context.Context, dockerCli command.Cli, cfg driver.InitConfig) (driver.Driver, error) {
contextStore := dockerCli.ContextStore()

var err error
var cc ClientConfig
cc, err = ctxkube.ConfigFromEndpoint(cfg.EndpointAddr, contextStore)
if err != nil {
// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
// try again with name="default".
// FIXME(@AkihiroSuda): n should retain real context name.
cc, err = ctxkube.ConfigFromEndpoint("default", contextStore)
if err != nil {
logrus.Error(err)
}
}
tryToUseKubeConfigInCluster := false
if cc == nil {
tryToUseKubeConfigInCluster = true
} else {
if _, err := cc.ClientConfig(); err != nil {
tryToUseKubeConfigInCluster = true
}
}
if tryToUseKubeConfigInCluster {
ccInCluster := ClientConfigInCluster{}
if _, err := ccInCluster.ClientConfig(); err == nil {
logrus.Debug("using kube config in cluster")
cc = ccInCluster
}
}
if cc == nil {
return nil, errors.Errorf("%s driver requires kubernetes API access", DriverName)
}

deploymentName, err := buildxNameToDeploymentName(cfg.Name)
if err != nil {
return nil, err
}
namespace, _, err := cfg.KubeClientConfig.Namespace()
namespace, _, err := cc.Namespace()
if err != nil {
return nil, errors.Wrap(err, "cannot determine Kubernetes namespace, specify manually")
}
restClientConfig, err := cfg.KubeClientConfig.ClientConfig()
restClientConfig, err := cc.ClientConfig()
if err != nil {
return nil, err
}
Expand Down
54 changes: 16 additions & 38 deletions driver/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,22 @@ package driver

import (
"context"
"os"
"sort"
"strings"
"sync"

"github.com/docker/cli/cli/command"
dockerclient "github.com/docker/docker/client"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/util/tracing/delegated"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"k8s.io/client-go/rest"
)

type Factory interface {
Name() string
Usage() string
Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, dialMeta map[string][]string) int
New(ctx context.Context, cfg InitConfig) (Driver, error)
New(ctx context.Context, dockerCli command.Cli, cfg InitConfig) (Driver, error)
AllowsInstances() bool
}

Expand All @@ -28,38 +26,18 @@ type BuildkitConfig struct {
// Rootless bool
}

type KubeClientConfig interface {
ClientConfig() (*rest.Config, error)
Namespace() (string, bool, error)
}

type KubeClientConfigInCluster struct{}

func (k KubeClientConfigInCluster) ClientConfig() (*rest.Config, error) {
return rest.InClusterConfig()
}

func (k KubeClientConfigInCluster) Namespace() (string, bool, error) {
namespace, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
if err != nil {
return "", false, err
}
return strings.TrimSpace(string(namespace)), true, nil
}

type InitConfig struct {
// TODO: This object needs updates to be generic for different drivers
Name string
EndpointAddr string
DockerAPI dockerclient.APIClient
KubeClientConfig KubeClientConfig
BuildkitdFlags []string
Files map[string][]byte
DriverOpts map[string]string
Auth Auth
Platforms []specs.Platform
ContextPathHash string // can be used for determining pods in the driver instance
DialMeta map[string][]string
Name string
EndpointAddr string
DockerAPI dockerclient.APIClient
BuildkitdFlags []string
Files map[string][]byte
DriverOpts map[string]string
Auth Auth
Platforms []specs.Platform
ContextPathHash string // can be used for determining pods in the driver instance
DialMeta map[string][]string
}

var drivers map[string]Factory
Expand All @@ -71,7 +49,7 @@ func Register(f Factory) {
drivers[f.Name()] = f
}

func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient, instanceRequired bool, dialMeta map[string][]string) (Factory, error) {
func GetDefaultFactory(ctx context.Context, dockerCli command.Cli, ep string, c dockerclient.APIClient, instanceRequired bool, dialMeta map[string][]string) (Factory, error) {
if len(drivers) == 0 {
return nil, errors.Errorf("no drivers available")
}
Expand Down Expand Up @@ -104,15 +82,15 @@ func GetFactory(name string, instanceRequired bool) (Factory, error) {
return nil, errors.Errorf("failed to find driver %q", name)
}

func GetDriver(ctx context.Context, f Factory, cfg InitConfig) (*DriverHandle, error) {
func GetDriver(ctx context.Context, f Factory, dockerCli command.Cli, cfg InitConfig) (*DriverHandle, error) {
if f == nil {
var err error
f, err = GetDefaultFactory(ctx, cfg.EndpointAddr, cfg.DockerAPI, false, cfg.DialMeta)
f, err = GetDefaultFactory(ctx, dockerCli, cfg.EndpointAddr, cfg.DockerAPI, false, cfg.DialMeta)
if err != nil {
return nil, err
}
}
d, err := f.New(ctx, cfg)
d, err := f.New(ctx, dockerCli, cfg)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 691c575

Please sign in to comment.