diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index 2931a5e611..8337a59abe 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -812,7 +812,7 @@ func provisionK8sOptions(sshClient libssh.Client, k8sClient k8s.K8sDynamicClient } if n.CNAO { - cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient) + cnaoOpt := cnao.NewCnaoOpt(k8sClient, sshClient, n.Multus) opts = append(opts, cnaoOpt) } diff --git a/cluster-provision/gocli/opts/cnao/cnao.go b/cluster-provision/gocli/opts/cnao/cnao.go index 2b1a053cfd..81a6f89b39 100644 --- a/cluster-provision/gocli/opts/cnao/cnao.go +++ b/cluster-provision/gocli/opts/cnao/cnao.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "path/filepath" + "regexp" "github.com/sirupsen/logrus" k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" @@ -16,14 +17,16 @@ import ( var f embed.FS type cnaoOpt struct { - client k8s.K8sDynamicClient - sshClient libssh.Client + client k8s.K8sDynamicClient + sshClient libssh.Client + multusEnabled bool } -func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client) *cnaoOpt { +func NewCnaoOpt(c k8s.K8sDynamicClient, sshClient libssh.Client, multusEnabled bool) *cnaoOpt { return &cnaoOpt{ - client: c, - sshClient: sshClient, + client: c, + sshClient: sshClient, + multusEnabled: multusEnabled, } } @@ -43,6 +46,12 @@ func (o *cnaoOpt) Exec() error { continue } + if path == "manifests/network-addons-config-example.cr.yaml" && o.multusEnabled { + re := regexp.MustCompile("(?m)[\r\n]+^.*multus.*$") + res := re.ReplaceAllString(string(yamlDoc), "") + yamlDoc = []byte(res) + } + obj, err := k8s.SerializeIntoObject(yamlDoc) if err != nil { logrus.Info(err.Error()) diff --git a/cluster-provision/gocli/opts/cnao/cnao_test.go b/cluster-provision/gocli/opts/cnao/cnao_test.go index f90eb02baa..4b2609f1d0 100644 --- a/cluster-provision/gocli/opts/cnao/cnao_test.go +++ b/cluster-provision/gocli/opts/cnao/cnao_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" + "k8s.io/apimachinery/pkg/runtime/schema" k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" ) @@ -27,16 +28,41 @@ var _ = Describe("CnaoOpt", func() { mockCtrl = gomock.NewController(GinkgoT()) client = k8s.NewTestClient() sshClient = kubevirtcimocks.NewMockSSHClient(mockCtrl) - opt = NewCnaoOpt(client, sshClient) }) AfterEach(func() { mockCtrl.Finish() }) - It("should execute CnaoOpt successfully", func() { + It("should execute create CNAO with Multus", func() { + opt = NewCnaoOpt(client, sshClient, false) sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s") - err := opt.Exec() + opt.Exec() + + obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io", + Version: "v1", + Kind: "NetworkAddonsConfig"}, "cluster", "") + Expect(err).NotTo(HaveOccurred()) + + spec, ok := obj.Object["spec"].(map[string]interface{}) + Expect(ok).To(Equal(true)) + Expect(spec).To(HaveKey("multus")) + Expect(spec).To(HaveKey("multusDynamicNetworks")) + }) + + It("should execute create CNAO without Multus", func() { + opt = NewCnaoOpt(client, sshClient, true) + sshClient.EXPECT().Command("kubectl --kubeconfig=/etc/kubernetes/admin.conf wait deployment -n cluster-network-addons cluster-network-addons-operator --for condition=Available --timeout=200s") + opt.Exec() + + obj, err := client.Get(schema.GroupVersionKind{Group: "networkaddonsoperator.network.kubevirt.io", + Version: "v1", + Kind: "NetworkAddonsConfig"}, "cluster", "") Expect(err).NotTo(HaveOccurred()) + + spec, ok := obj.Object["spec"].(map[string]interface{}) + Expect(ok).To(Equal(true)) + Expect(spec).NotTo(HaveKey("multus")) + Expect(spec).NotTo(HaveKey("multusDynamicNetworks")) }) }) diff --git a/cluster-provision/gocli/opts/cnao/manifests/crd.yaml b/cluster-provision/gocli/opts/cnao/manifests/crd.yaml index 9f84c50f09..e173349389 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/crd.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/crd.yaml @@ -59,6 +59,10 @@ spec: description: NameServerIp defines the name server IP type: string type: object + kubevirtIpamController: + description: KubevirtIpamController plugin allows to support IPAM + for secondary networks + type: object linuxBridge: description: LinuxBridge plugin allows users to create a bridge and add the host and the container to it @@ -1521,20 +1525,6 @@ spec: - ECDHE-RSA-CHACHA20-POLY1305 - DHE-RSA-AES128-GCM-SHA256 - DHE-RSA-AES256-GCM-SHA384 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - ECDHE-ECDSA-AES128-GCM-SHA256 - - ECDHE-RSA-AES128-GCM-SHA256 - - ECDHE-ECDSA-AES256-GCM-SHA384 - - ECDHE-RSA-AES256-GCM-SHA384 - - ECDHE-ECDSA-CHACHA20-POLY1305 - - ECDHE-RSA-CHACHA20-POLY1305 - - DHE-RSA-AES128-GCM-SHA256 - - DHE-RSA-AES256-GCM-SHA384 - DHE-RSA-CHACHA20-POLY1305 - ECDHE-ECDSA-AES128-SHA256 - ECDHE-RSA-AES128-SHA256 @@ -1706,6 +1696,10 @@ spec: description: NameServerIp defines the name server IP type: string type: object + kubevirtIpamController: + description: KubevirtIpamController plugin allows to support IPAM + for secondary networks + type: object linuxBridge: description: LinuxBridge plugin allows users to create a bridge and add the host and the container to it @@ -3168,20 +3162,6 @@ spec: - ECDHE-RSA-CHACHA20-POLY1305 - DHE-RSA-AES128-GCM-SHA256 - DHE-RSA-AES256-GCM-SHA384 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - ECDHE-ECDSA-AES128-GCM-SHA256 - - ECDHE-RSA-AES128-GCM-SHA256 - - ECDHE-ECDSA-AES256-GCM-SHA384 - - ECDHE-RSA-AES256-GCM-SHA384 - - ECDHE-ECDSA-CHACHA20-POLY1305 - - ECDHE-RSA-CHACHA20-POLY1305 - - DHE-RSA-AES128-GCM-SHA256 - - DHE-RSA-AES256-GCM-SHA384 - DHE-RSA-CHACHA20-POLY1305 - ECDHE-ECDSA-AES128-SHA256 - ECDHE-RSA-AES128-SHA256 diff --git a/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml b/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml index ebe63fda20..972335a37f 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/namespace.yaml @@ -1,7 +1,8 @@ +--- apiVersion: v1 kind: Namespace metadata: name: cluster-network-addons labels: name: cluster-network-addons - openshift.io/cluster-monitoring: "true" \ No newline at end of file + openshift.io/cluster-monitoring: "true" diff --git a/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml b/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml index 0c5662837e..718853d3a6 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/network-addons-config-example.cr.yaml @@ -1,3 +1,4 @@ +--- apiVersion: networkaddonsoperator.network.kubevirt.io/v1 kind: NetworkAddonsConfig metadata: @@ -10,4 +11,3 @@ spec: macvtap: {} multus: {} multusDynamicNetworks: {} - ovs: {} \ No newline at end of file diff --git a/cluster-provision/gocli/opts/cnao/manifests/operator.yaml b/cluster-provision/gocli/opts/cnao/manifests/operator.yaml index fd7bb41929..6ee0724822 100644 --- a/cluster-provision/gocli/opts/cnao/manifests/operator.yaml +++ b/cluster-provision/gocli/opts/cnao/manifests/operator.yaml @@ -120,6 +120,24 @@ rules: - services verbs: - delete +- apiGroups: + - k8s.cni.cncf.io + resources: + - ipamclaims + verbs: + - get + - list + - watch + - create + - update +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -144,12 +162,6 @@ rules: - create - patch - update -- apiGroups: - - k8s.cni.cncf.io - resources: - - '*' - verbs: - - '*' - apiGroups: - "" resources: @@ -238,6 +250,12 @@ rules: - get - list - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - '*' + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 @@ -329,6 +347,34 @@ rules: verbs: - get - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - patch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - get + - create + - update + - delete --- apiVersion: rbac.authorization.k8s.io/v1 @@ -349,7 +395,7 @@ apiVersion: apps/v1 kind: Deployment metadata: annotations: - networkaddonsoperator.network.kubevirt.io/version: 0.87.0 + networkaddonsoperator.network.kubevirt.io/version: 0.94.1 labels: prometheus.cnao.io: "true" name: cluster-network-addons-operator @@ -375,29 +421,31 @@ spec: - name: MULTUS_IMAGE value: ghcr.io/k8snetworkplumbingwg/multus-cni@sha256:3fbcc32bd4e4d15bd93c96def784a229cd84cca27942bf4858b581f31c97ee02 - name: MULTUS_DYNAMIC_NETWORKS_CONTROLLER_IMAGE - value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:dee1979d92f0a31598a6e3569ac7004be7d29e7ca9e31db23753ef263110dc04 + value: ghcr.io/k8snetworkplumbingwg/multus-dynamic-networks-controller@sha256:83b460502671fb4f34116363a1a39b2ddfc9d14a920ee0a6413bfc3bd0580404 - name: LINUX_BRIDGE_IMAGE - value: quay.io/kubevirt/cni-default-plugins@sha256:825e3f9fec1996c54a52cec806154945b38f76476b160d554c36e38dfffe5e61 + value: quay.io/kubevirt/cni-default-plugins@sha256:0c354fa9d695b8cab97b459e8afea2f7662407a987e83f6f6f1a8af4b45726be - name: LINUX_BRIDGE_MARKER_IMAGE - value: quay.io/kubevirt/bridge-marker@sha256:5d24c6d1ecb0556896b7b81c7e5260b54173858425777b7a84df8a706c07e6d2 + value: quay.io/kubevirt/bridge-marker@sha256:bba066e3b5ff3fb8c5e20861fe8abe51e3c9b50ad6ce3b2616af9cb5479a06d0 - name: OVS_CNI_IMAGE - value: quay.io/kubevirt/ovs-cni-plugin@sha256:5f7290e2294255ab2547c3b4bf48cc2d75531ec5a43e600366e9b2719bef983f + value: quay.io/kubevirt/ovs-cni-plugin@sha256:e16ac74343da21abb8fb668ce71e728053d00503a992dae2164b9e94a280113e - name: KUBEMACPOOL_IMAGE - value: quay.io/kubevirt/kubemacpool@sha256:afba7d0c4a95d2d4924f6ee6ef16bbe59117877383819057f01809150829cb0c + value: quay.io/kubevirt/kubemacpool@sha256:20e156be33e6d3692c456081acbb91a4349f94de448f7d1f1cddd0228931b31f - name: MACVTAP_CNI_IMAGE - value: quay.io/kubevirt/macvtap-cni@sha256:434420511e09b2b5ede785a2c9062b6658ffbc26fbdd4629ce06110f9039c600 + value: quay.io/kubevirt/macvtap-cni@sha256:850b89343ace7c7ea6b18dd8e11964613974e9d1f7377af03854d407fb15230a - name: KUBE_RBAC_PROXY_IMAGE - value: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901 + value: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8 - name: KUBE_SECONDARY_DNS_IMAGE - value: ghcr.io/kubevirt/kubesecondarydns@sha256:77132adb5f840ceb0aadd408731a5c8b01a4b427a78084ab5e4e9b961195cb02 + value: ghcr.io/kubevirt/kubesecondarydns@sha256:6268d84154e2483fbce8c1adacbdaf6f0839117b2d48d9fa4687cc8f76bd5130 - name: CORE_DNS_IMAGE value: registry.k8s.io/coredns/coredns@sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e + - name: KUBEVIRT_IPAM_CONTROLLER_IMAGE + value: ghcr.io/kubevirt/ipam-controller@sha256:f272eaf82c9e4fcd7fdee5d9031afba8019cff2dc842ca3e36be49de28083fee - name: OPERATOR_IMAGE - value: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0 + value: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1 - name: OPERATOR_NAME value: cluster-network-addons-operator - name: OPERATOR_VERSION - value: 0.87.0 + value: 0.94.1 - name: OPERATOR_NAMESPACE valueFrom: fieldRef: @@ -416,8 +464,8 @@ spec: - name: MONITORING_SERVICE_ACCOUNT value: prometheus-k8s - name: RUNBOOK_URL_TEMPLATE - value: https://kubevirt.io/monitoring/runbooks/ - image: quay.io/kubevirt/cluster-network-addons-operator:v0.87.0 + value: https://kubevirt.io/monitoring/runbooks/%s + image: quay.io/kubevirt/cluster-network-addons-operator:v0.94.1 imagePullPolicy: Always name: cluster-network-addons-operator resources: @@ -433,7 +481,7 @@ spec: - --logtostderr - --secure-listen-address=:8443 - --upstream=http://127.0.0.1:8080 - image: quay.io/openshift/origin-kube-rbac-proxy@sha256:baedb268ac66456018fb30af395bb3d69af5fff3252ff5d549f0231b1ebb6901 + image: quay.io/openshift/origin-kube-rbac-proxy@sha256:e2def4213ec0657e72eb790ae8a115511d5b8f164a62d3568d2f1bff189917e8 imagePullPolicy: Always name: kube-rbac-proxy ports: @@ -455,4 +503,4 @@ spec: runAsNonRoot: true seccompProfile: type: RuntimeDefault - serviceAccountName: cluster-network-addons-operator \ No newline at end of file + serviceAccountName: cluster-network-addons-operator diff --git a/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml b/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml index d6bad14670..7e50e2bc85 100644 --- a/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml +++ b/cluster-provision/gocli/opts/istio/manifests/istio-operator-with-cnao.cr.yaml @@ -22,7 +22,3 @@ spec: - istio-system - kube-system logLevel: debug - cniConfFileName: "istio-cni.conf" - sidecarInjectorWebhook: - injectedAnnotations: - "k8s.v1.cni.cncf.io/networks": istio-cni \ No newline at end of file diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index 8125112ae8..32c4ede425 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -14,7 +14,16 @@ function deploy_kwok() { fi } - +# copy_istio_cni_conf_files copy the generated Istio CNI net conf file +# (at '/etc/cni/multus/net.d/') to where Multus expect CNI net conf files ('/etc/cni/net.d/') +function copy_istio_cni_conf_files() { + if [ "$KUBEVIRT_DEPLOY_ISTIO" == "true" ] && [ "$KUBEVIRT_WITH_CNAO" == "true" ]; then + for nodeNum in $(seq -f "%02g" 1 $KUBEVIRT_NUM_NODES); do + $ssh node${nodeNum} -- "until ls /etc/cni/multus > /dev/null 2>&1; do sleep 1; done" + $ssh node${nodeNum} -- sudo cp -uv /etc/cni/multus/net.d/*istio*.conf /etc/cni/net.d/ + done + fi +} # configure Prometheus to select kubevirt prometheusrules function configure_prometheus() { @@ -95,4 +104,9 @@ function up() { echo "Waiting for cluster components..." sleep 5 done + + # FIXME: remove 'copy_istio_cni_conf_files()' as soon as [1] and [2] are resolved + # [1] https://github.com/kubevirt/kubevirtci/issues/906 + # [2] https://github.com/k8snetworkplumbingwg/multus-cni/issues/982 + copy_istio_cni_conf_files }