diff --git a/hack/cluster.sh b/hack/cluster.sh index 3a7fa27a..d7b7a698 100755 --- a/hack/cluster.sh +++ b/hack/cluster.sh @@ -3,7 +3,7 @@ set -xe SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -KIND_ARGS="${KIND_ARGS:--ic -ikv -i6 -mne}" +KIND_ARGS="${KIND_ARGS:--ic -ikv -i6 -mne -nse}" OUTPUT_DIR=${OUTPUT_DIR:-${SCRIPT_DIR}/../.output} diff --git a/test/e2e/persistentips_test.go b/test/e2e/persistentips_test.go index 081f31c0..738f8d53 100644 --- a/test/e2e/persistentips_test.go +++ b/test/e2e/persistentips_test.go @@ -40,7 +40,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("Persistent IPs", func() { +const ( + secondaryLogicalNetworkInterfaceName = "multus" + nadName = "l2-net-attach-def" +) + +const ( + rolePrimary = "primary" + roleSecondary = "secondary" +) + +type testParams struct { + role string + ipsFrom func(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) + vmi func(namespace string) *kubevirtv1.VirtualMachineInstance +} + +var _ = DescribeTableSubtree("Persistent IPs", func(params testParams) { var failureCount int = 0 JustAfterEach(func() { if CurrentSpecReport().Failed() { @@ -57,12 +73,12 @@ var _ = Describe("Persistent IPs", func() { When("network attachment definition created with allowPersistentIPs=true", func() { var ( - td testenv.TestData - networkInterfaceName = "multus" - vm *kubevirtv1.VirtualMachine - vmi *kubevirtv1.VirtualMachineInstance - nad *nadv1.NetworkAttachmentDefinition + td testenv.TestData + vm *kubevirtv1.VirtualMachine + vmi *kubevirtv1.VirtualMachineInstance + nad *nadv1.NetworkAttachmentDefinition ) + BeforeEach(func() { td = testenv.GenerateTestData() td.SetUp() @@ -70,8 +86,8 @@ var _ = Describe("Persistent IPs", func() { td.TearDown() }) - nad = testenv.GenerateLayer2WithSubnetNAD(td.Namespace) - vmi = testenv.GenerateAlpineWithMultusVMI(td.Namespace, networkInterfaceName, nad.Name) + nad = testenv.GenerateLayer2WithSubnetNAD(nadName, td.Namespace, params.role) + vmi = params.vmi(td.Namespace) vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning()) By("Create NetworkAttachmentDefinition") @@ -94,14 +110,14 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { - vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeMigration, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) + Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name) testenv.CheckLiveMigrationSucceeded(td.Namespace, vm.Name) @@ -112,8 +128,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration))) - + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VM deletion", func() { @@ -171,7 +186,10 @@ var _ = Describe("Persistent IPs", func() { }) It("should keep ips after restart", func() { - vmiIPsBeforeRestart := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeRestart, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) + Expect(vmiIPsBeforeRestart).NotTo(BeEmpty()) vmiUUIDBeforeRestart := vmi.UID By("Re-starting the VM") @@ -190,7 +208,7 @@ var _ = Describe("Persistent IPs", func() { WithTimeout(5 * time.Minute). Should(testenv.ContainConditionVMIReady()) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeRestart))) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeRestart))) }) }) @@ -217,9 +235,9 @@ var _ = Describe("Persistent IPs", func() { ShouldNot(BeEmpty()) Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + ips, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) + Expect(ips).NotTo(BeEmpty()) }) It("should garbage collect IPAMClaims after VM foreground deletion, only after VMI is gone", func() { @@ -260,20 +278,19 @@ var _ = Describe("Persistent IPs", func() { WithPolling(time.Second). ShouldNot(BeEmpty()) - Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) - - Expect(vmi.Status.Interfaces).NotTo(BeEmpty()) - Expect(vmi.Status.Interfaces[0].IPs).NotTo(BeEmpty()) + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, Not(BeEmpty()))) }) It("should keep ips after live migration", func() { - vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs + Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) + vmiIPsBeforeMigration, err := params.ipsFrom(vmi) + Expect(err).NotTo(HaveOccurred()) + Expect(vmiIPsBeforeMigration).NotTo(BeEmpty()) testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name) testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name) - Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration))) - + Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPs(params.ipsFrom, ConsistOf(vmiIPsBeforeMigration))) }) It("should garbage collect IPAMClaims after VMI deletion", func() { @@ -294,7 +311,20 @@ var _ = Describe("Persistent IPs", func() { }) }) -}) +}, + Entry("secondary interfaces", + testParams{ + role: roleSecondary, + ipsFrom: secondaryNetworkVMIStatusIPs, + vmi: vmiWithMultus, + }), + Entry("primary UDN", + testParams{ + role: rolePrimary, + ipsFrom: defaultNetworkStatusAnnotationIPs, + vmi: vmiWithPasst, + }), +) func foregroundDeleteOptions() *client.DeleteOptions { foreground := metav1.DeletePropagationForeground @@ -311,3 +341,67 @@ func removeFinalizersPatch() ([]byte, error) { } return json.Marshal(patch) } + +func secondaryNetworkVMIStatusIPs(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) { + return testenv.GetIPsFromVMIStatus(vmi, secondaryLogicalNetworkInterfaceName), nil +} + +func defaultNetworkStatusAnnotationIPs(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error) { + defNetworkStatus, err := testenv.DefaultNetworkStatus(vmi) + if err != nil { + return nil, err + } + + return defNetworkStatus.IPs, nil +} + +func vmiWithMultus(namespace string) *kubevirtv1.VirtualMachineInstance { + interfaceName := secondaryLogicalNetworkInterfaceName + return testenv.NewVirtualMachineInstance( + namespace, + testenv.WithMemory("128Mi"), + testenv.WithInterface(kubevirtv1.Interface{ + Name: interfaceName, + InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ + Bridge: &kubevirtv1.InterfaceBridge{}, + }, + }), + testenv.WithNetwork(kubevirtv1.Network{ + + Name: interfaceName, + NetworkSource: kubevirtv1.NetworkSource{ + Multus: &kubevirtv1.MultusNetwork{ + NetworkName: nadName, + }, + }, + }), + ) +} + +func vmiWithPasst(namespace string) *kubevirtv1.VirtualMachineInstance { + const ( + interfaceName = "passtnet" + cloudInitNetworkData = ` +version: 2 +ethernets: + eth0: + dhcp4: true` + ) + return testenv.NewVirtualMachineInstance( + namespace, + testenv.WithMemory("2048Mi"), + testenv.WithInterface(kubevirtv1.Interface{ + Name: interfaceName, + Binding: &kubevirtv1.PluginBinding{ + Name: "passt", + }, + }), + testenv.WithNetwork(kubevirtv1.Network{ + Name: interfaceName, + NetworkSource: kubevirtv1.NetworkSource{ + Pod: &kubevirtv1.PodNetwork{}, + }, + }), + testenv.WithCloudInitNoCloudVolume(cloudInitNetworkData), + ) +} diff --git a/test/env/generate.go b/test/env/generate.go index 66813054..c5440c70 100644 --- a/test/env/generate.go +++ b/test/env/generate.go @@ -13,9 +13,8 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" ) -func GenerateLayer2WithSubnetNAD(namespace string) *nadv1.NetworkAttachmentDefinition { +func GenerateLayer2WithSubnetNAD(nadName, namespace, role string) *nadv1.NetworkAttachmentDefinition { networkName := "l2" - nadName := RandomName(networkName, 16) return &nadv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -30,26 +29,24 @@ func GenerateLayer2WithSubnetNAD(namespace string) *nadv1.NetworkAttachmentDefin "topology": "layer2", "subnets": "10.100.200.0/24", "netAttachDefName": "%[1]s/%[2]s", + "role": "%[4]s", "allowPersistentIPs": true } -`, namespace, nadName, networkName), +`, namespace, nadName, networkName, role), }, } } -func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) *kubevirtv1.VirtualMachineInstance { - return &kubevirtv1.VirtualMachineInstance{ +type VMIOption func(vmi *kubevirtv1.VirtualMachineInstance) + +func NewVirtualMachineInstance(namespace string, opts ...VMIOption) *kubevirtv1.VirtualMachineInstance { + vmi := &kubevirtv1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: RandomName("alpine", 16), }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ Domain: kubevirtv1.DomainSpec{ - Resources: kubevirtv1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - }, Devices: kubevirtv1.Devices{ Disks: []kubevirtv1.Disk{ { @@ -61,26 +58,10 @@ func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) * Name: "containerdisk", }, }, - Interfaces: []kubevirtv1.Interface{ - { - Name: interfaceName, - InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{ - Bridge: &kubevirtv1.InterfaceBridge{}, - }, - }, - }, - }, - }, - Networks: []kubevirtv1.Network{ - { - Name: interfaceName, - NetworkSource: kubevirtv1.NetworkSource{ - Multus: &kubevirtv1.MultusNetwork{ - NetworkName: networkName, - }, - }, + Interfaces: []kubevirtv1.Interface{}, }, }, + Networks: []kubevirtv1.Network{}, TerminationGracePeriodSeconds: pointer.Int64(5), Volumes: []kubevirtv1.Volume{ { @@ -94,6 +75,45 @@ func GenerateAlpineWithMultusVMI(namespace, interfaceName, networkName string) * }, }, } + + for _, f := range opts { + f(vmi) + } + + return vmi +} + +func WithMemory(memory string) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Domain.Resources.Requests = corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(memory), + } + } +} + +func WithInterface(iface kubevirtv1.Interface) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Domain.Devices.Interfaces = append(vmi.Spec.Domain.Devices.Interfaces, iface) + } +} + +func WithNetwork(network kubevirtv1.Network) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Networks = append(vmi.Spec.Networks, network) + } +} + +func WithCloudInitNoCloudVolume(cloudInitNetworkData string) VMIOption { + return func(vmi *kubevirtv1.VirtualMachineInstance) { + vmi.Spec.Volumes = append(vmi.Spec.Volumes, kubevirtv1.Volume{ + Name: "cloudinitdisk", + VolumeSource: kubevirtv1.VolumeSource{ + CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ + NetworkData: cloudInitNetworkData, + }, + }, + }) + } } type VMOption func(vm *kubevirtv1.VirtualMachine) diff --git a/test/env/getter.go b/test/env/getter.go index 7c620cc9..4f6f52b6 100644 --- a/test/env/getter.go +++ b/test/env/getter.go @@ -2,10 +2,16 @@ package env import ( "context" + "encoding/json" + "fmt" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - kubevirtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + kubevirtv1 "kubevirt.io/api/core/v1" ) // ThisVMI fetches the latest state of the VirtualMachineInstance. If the object does not exist, nil is returned. @@ -28,3 +34,92 @@ func ThisVMReadiness(vm *kubevirtv1.VirtualMachine) func() (bool, error) { return vm.Status.Ready, nil } } + +func lookupInterfaceStatusByName(interfaces []kubevirtv1.VirtualMachineInstanceNetworkInterface, name string) *kubevirtv1.VirtualMachineInstanceNetworkInterface { + for index := range interfaces { + if interfaces[index].Name == name { + return &interfaces[index] + } + } + return nil +} + +func GetIPsFromVMIStatus(vmi *kubevirtv1.VirtualMachineInstance, networkInterfaceName string) []string { + ifaceStatus := lookupInterfaceStatusByName(vmi.Status.Interfaces, networkInterfaceName) + if ifaceStatus == nil { + return nil + } + return ifaceStatus.IPs +} + +func virtualMachineInstancePod(vmi *kubevirtv1.VirtualMachineInstance) (*corev1.Pod, error) { + pod, err := lookupPodBySelector(vmi.Namespace, vmiLabelSelector(vmi), vmiFieldSelector(vmi)) + if err != nil { + return nil, fmt.Errorf("failed to find pod for VMI %s (%s)", vmi.Name, string(vmi.GetUID())) + } + return pod, nil +} + +func lookupPodBySelector(namespace string, labelSelector, fieldSelector map[string]string) (*corev1.Pod, error) { + pods := &corev1.PodList{} + err := Client.List( + context.Background(), + pods, + client.InNamespace(namespace), + client.MatchingLabels(labelSelector), + client.MatchingFields(fieldSelector)) + if err != nil { + return nil, err + } + + if len(pods.Items) == 0 { + return nil, fmt.Errorf("failed to lookup pod with labels %v, fields %v in namespace %s", labelSelector, fieldSelector, namespace) + } + + return &pods.Items[0], nil +} + +func vmiLabelSelector(vmi *kubevirtv1.VirtualMachineInstance) map[string]string { + return map[string]string{kubevirtv1.CreatedByLabel: string(vmi.GetUID())} +} + +func vmiFieldSelector(vmi *kubevirtv1.VirtualMachineInstance) map[string]string { + fieldSelectors := map[string]string{} + if vmi.Status.Phase == kubevirtv1.Running { + const podPhase = "status.phase" + fieldSelectors[podPhase] = string(corev1.PodRunning) + } + return fieldSelectors +} + +func parsePodNetworkStatusAnnotation(podNetStatus string) ([]nadv1.NetworkStatus, error) { + if len(podNetStatus) == 0 { + return nil, fmt.Errorf("network status annotation not found") + } + + var netStatus []nadv1.NetworkStatus + if err := json.Unmarshal([]byte(podNetStatus), &netStatus); err != nil { + return nil, err + } + + return netStatus, nil +} + +func DefaultNetworkStatus(vmi *kubevirtv1.VirtualMachineInstance) (*nadv1.NetworkStatus, error) { + virtLauncherPod, err := virtualMachineInstancePod(vmi) + if err != nil { + return nil, err + } + + netStatuses, err := parsePodNetworkStatusAnnotation(virtLauncherPod.Annotations[nadv1.NetworkStatusAnnot]) + if err != nil { + return nil, err + } + + for _, netStatus := range netStatuses { + if netStatus.Default { + return &netStatus, nil + } + } + return nil, fmt.Errorf("primary IPs not found") +} diff --git a/test/env/matcher.go b/test/env/matcher.go index ec789ce6..c7654b54 100644 --- a/test/env/matcher.go +++ b/test/env/matcher.go @@ -28,37 +28,23 @@ func IPAMClaimsFromNamespace(namespace string) func() ([]ipamclaimsv1alpha1.IPAM } } -func lookupInterfaceStatusByName(interfaces []kubevirtv1.VirtualMachineInstanceNetworkInterface, name string) *kubevirtv1.VirtualMachineInstanceNetworkInterface { - for index := range interfaces { - if interfaces[index].Name == name { - return &interfaces[index] - } - } - return nil -} - -func VMIStatusInterfaces(vmi *kubevirtv1.VirtualMachineInstance) []kubevirtv1.VirtualMachineInstanceNetworkInterface { - return vmi.Status.Interfaces -} func vmiStatusConditions(vmi *kubevirtv1.VirtualMachineInstance) []kubevirtv1.VirtualMachineInstanceCondition { return vmi.Status.Conditions } -func interfaceIPs(networkInterface *kubevirtv1.VirtualMachineInstanceNetworkInterface) []string { - if networkInterface == nil { - return nil - } - return networkInterface.IPs +type IPResult struct { + IPs []string + Err error } -func MatchIPsAtInterfaceByName(interfaceName string, ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { - return WithTransform( - func(vmi *kubevirtv1.VirtualMachineInstance) *kubevirtv1.VirtualMachineInstanceNetworkInterface { - return lookupInterfaceStatusByName(vmi.Status.Interfaces, interfaceName) - }, - SatisfyAll( - Not(BeNil()), - WithTransform(interfaceIPs, ipsMatcher))) +func MatchIPs(getIPsFunc func(vmi *kubevirtv1.VirtualMachineInstance) ([]string, error), ipsMatcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { + return WithTransform(func(vmi *kubevirtv1.VirtualMachineInstance) IPResult { + ips, err := getIPsFunc(vmi) + return IPResult{IPs: ips, Err: err} + }, SatisfyAll( + WithTransform(func(result IPResult) error { return result.Err }, Succeed()), + WithTransform(func(result IPResult) []string { return result.IPs }, ipsMatcher), + )) } func BeRestarted(oldUID types.UID) gomegatypes.GomegaMatcher {