Skip to content

Commit

Permalink
Read configuration for the app directly from the k8s API
Browse files Browse the repository at this point in the history 
This is required to avoid possible issues caused by
ConfigMap cache in kubelet.

Signed-off-by: Yury Kulazhenkov <[email protected]>
  • Loading branch information
@ykulazhenkov
ykulazhenkov committed Nov 23, 2023
1 parent a838261 commit ebe66b0
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 96 deletions.
56 changes: 44 additions & 12 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,29 @@
# network-operator-init-container
Init container for NVIDIA Network Operator

The network-operator-init-container container has two required command line arguments:
## Configuration
The network-operator-init-container container has following required command line arguments:

- `--config` path to the configuration file
- `--configmap-name` name of the configmap with configuration for the app
- `--configmap-namespace` namespace of the configmap with configuration for the app
- `--node-name` name of the k8s node on which this app runs

The configuration file should be in JSON format:
The ConfigMap should include configuration in JSON format:

```
{
"safeDriverLoad": {
"enable": true,
"annotation": "some-annotation"
}
}
apiVersion: v1
kind: ConfigMap
metadata:
name: ofed-init-container-config
namespace: default
data:
config.json: |-
{
"safeDriverLoad": {
"enable": true,
"annotation": "some-annotation"
}
}
```

- `safeDriverLoad` - contains settings related to safeDriverLoad feature
Expand All @@ -28,6 +37,25 @@ The container exits with code 0 when the annotation is removed from the Node obj

If `safeDriverLoad` feature is disabled then the container will immediately exit with code 0.

### Required permissions

```
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: network-operator-init-container
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "patch", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
```

## Command line arguments

```
NVIDIA Network Operator init container
Expand All @@ -36,8 +64,12 @@ Usage:
Config flags:
--config string
path to the configuration file
--configmap-key string
key inside the configmap with configuration for the app (default "config.json")
--configmap-name string
name of the configmap with configuration for the app
--configmap-namespace string
namespace of the configmap with configuration for the app
--node-name string
name of the k8s node on which this app runs
Expand Down Expand Up @@ -70,4 +102,4 @@ Kubernetes flags:
--kubeconfig string
Paths to a kubeconfig. Only required if out-of-cluster.
```
```
38 changes: 25 additions & 13 deletions cmd/network-operator-init-container/app/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,6 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
"Options", opts, "Version", version.GetVersionString())
ctrl.SetLogger(logger)

initContCfg, err := configPgk.FromFile(opts.ConfigPath)
if err != nil {
logger.Error(err, "failed to read configuration")
return err
}
logger.Info("network-operator-init-container configuration", "config", initContCfg.String())

if !initContCfg.SafeDriverLoad.Enable {
logger.Info("safe driver loading is disabled, exit")
return nil
}

mgr, err := ctrl.NewManager(config, ctrl.Options{
Metrics: metricsserver.Options{BindAddress: "0"},
Cache: cache.Options{
Expand All @@ -117,7 +105,7 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
fmt.Sprintf("metadata.name=%s", opts.NodeName))}}},
})
if err != nil {
logger.Error(err, "unable to start manager")
logger.Error(err, "unable to create manager")
return err
}

Expand All @@ -128,6 +116,30 @@ func RunNetworkOperatorInitContainer(ctx context.Context, config *rest.Config, o
return err
}

confConfigMap := &corev1.ConfigMap{}

err = k8sClient.Get(ctx, client.ObjectKey{
Name: opts.ConfigMapName,
Namespace: opts.ConfigMapNamespace,
}, confConfigMap)

if err != nil {
logger.Error(err, "failed to read config map with configuration")
return err
}

initContCfg, err := configPgk.Load(confConfigMap.Data[opts.ConfigMapKey])
if err != nil {
logger.Error(err, "failed to read configuration")
return err
}
logger.Info("network-operator-init-container configuration", "config", initContCfg.String())

if !initContCfg.SafeDriverLoad.Enable {
logger.Info("safe driver loading is disabled, exit")
return nil
}

errCh := make(chan error, 1)

if err = (&NodeReconciler{
Expand Down
82 changes: 52 additions & 30 deletions cmd/network-operator-init-container/app/app_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@
package app_test

import (
"context"
"fmt"
"os"
"path/filepath"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -34,8 +34,11 @@ import (
)

const (
testNodeName = "node1"
testAnnotation = "foo.bar/spam"
testConfigMapName = "test"
testConfigMapNamespace = "default"
testConfigMapKey = "conf"
testNodeName = "node1"
testAnnotation = "foo.bar/spam"
)

func createNode(name string) *corev1.Node {
Expand All @@ -44,44 +47,67 @@ func createNode(name string) *corev1.Node {
return node
}

func createConfig(path string, cfg configPgk.Config) {
func createConfig(cfg configPgk.Config) {
data, err := json.Marshal(cfg)
ExpectWithOffset(1, err).NotTo(HaveOccurred())
ExpectWithOffset(1, os.WriteFile(path, data, 0x744))
err = k8sClient.Create(ctx, &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: testConfigMapName, Namespace: testConfigMapNamespace},
Data: map[string]string{testConfigMapKey: string(data)},
})
ExpectWithOffset(1, err).NotTo(HaveOccurred())
}

func newOpts() *options.Options {
return &options.Options{
ConfigMapName: testConfigMapName,
ConfigMapNamespace: testConfigMapNamespace,
ConfigMapKey: testConfigMapKey,
}
}

var _ = Describe("Init container", func() {
var (
configPath string
testCtx context.Context
testCFunc context.CancelFunc
)

BeforeEach(func() {
configPath = filepath.Join(GinkgoT().TempDir(), "config")
testCtx, testCFunc = context.WithCancel(ctx)
})

AfterEach(func() {
err := k8sClient.Delete(ctx, &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: testConfigMapName, Namespace: testConfigMapNamespace},
})
if !apiErrors.IsNotFound(err) {
Expect(err).NotTo(HaveOccurred())
}
testCFunc()
})
It("Succeed", func() {
testDone := make(chan interface{})
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
node := &corev1.Node{}
Eventually(func(g Gomega) {
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testNodeName}, node)).NotTo(HaveOccurred())
g.Expect(k8sClient.Get(testCtx, types.NamespacedName{Name: testNodeName}, node)).NotTo(HaveOccurred())
g.Expect(node.GetAnnotations()[testAnnotation]).NotTo(BeEmpty())
}, 30, 1).Should(Succeed())
// remove annotation
Expect(k8sClient.Patch(ctx, node, client.RawPatch(
Expect(k8sClient.Patch(testCtx, node, client.RawPatch(
types.MergePatchType, []byte(
fmt.Sprintf(`{"metadata":{"annotations":{%q: null}}}`,
testAnnotation))))).NotTo(HaveOccurred())
Expand All @@ -95,17 +121,16 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = "unknown-node"
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand All @@ -118,20 +143,19 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: true,
Annotation: testAnnotation,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
cFunc()
testCFunc()
Eventually(appExit, 30, 1).Should(BeClosed())
Expect(err).To(HaveOccurred())
}()
Expand All @@ -142,13 +166,12 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = "unknown-node"
opts.ConfigPath = configPath
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand All @@ -161,16 +184,15 @@ var _ = Describe("Init container", func() {
go func() {
defer close(testDone)
defer GinkgoRecover()
opts := options.New()
opts := newOpts()
opts.NodeName = testNodeName
opts.ConfigPath = configPath
createConfig(configPath, configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
createConfig(configPgk.Config{SafeDriverLoad: configPgk.SafeDriverLoadConfig{
Enable: false,
}})
var err error
appExit := make(chan interface{})
go func() {
err = app.RunNetworkOperatorInitContainer(ctx, cfg, opts)
err = app.RunNetworkOperatorInitContainer(testCtx, cfg, opts)
close(appExit)
}()
Eventually(appExit, 30, 1).Should(BeClosed())
Expand Down
28 changes: 21 additions & 7 deletions cmd/network-operator-init-container/app/options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,24 @@ func New() *Options {

// Options contains application options
type Options struct {
NodeName string
ConfigPath string
LogConfig *logsapi.LoggingConfiguration
NodeName string
ConfigMapName string
ConfigMapNamespace string
ConfigMapKey string
LogConfig *logsapi.LoggingConfiguration
}

// AddNamedFlagSets returns FlagSet for Options
func (o *Options) AddNamedFlagSets(sharedFS *cliflag.NamedFlagSets) {
configFS := sharedFS.FlagSet("Config")
configFS.StringVar(&o.NodeName, "node-name", "",
"name of the k8s node on which this app runs")
configFS.StringVar(&o.ConfigPath, "config", "",
"path to the configuration file")
configFS.StringVar(&o.ConfigMapName, "configmap-name", "",
"name of the configmap with configuration for the app")
configFS.StringVar(&o.ConfigMapNamespace, "configmap-namespace", "",
"namespace of the configmap with configuration for the app")
configFS.StringVar(&o.ConfigMapKey, "configmap-key", "config.json",
"key inside the configmap with configuration for the app")

logFS := sharedFS.FlagSet("Logging")
logsapi.AddFlags(o.LogConfig, logFS)
Expand All @@ -68,8 +74,16 @@ func (o *Options) Validate() error {
return fmt.Errorf("node-name is required parameter")
}

if o.ConfigPath == "" {
return fmt.Errorf("config is required parameter")
if o.ConfigMapName == "" {
return fmt.Errorf("configmap-name is required parameter")
}

if o.ConfigMapNamespace == "" {
return fmt.Errorf("configmap-namespace is required parameter")
}

if o.ConfigMapKey == "" {
return fmt.Errorf("configmap-key is required parameter")
}

if err = logsapi.ValidateAndApply(o.LogConfig, nil); err != nil {
Expand Down
Loading

0 comments on commit ebe66b0

Please sign in to comment.