From f69e11785eb7771a5541426f04606162d08d10fa Mon Sep 17 00:00:00 2001 From: Grant Spence Date: Mon, 19 Aug 2024 21:43:38 -0400 Subject: [PATCH] WIP DO NOT MERGE: DNS Propagation Test --- test/e2e/all_test.go | 205 ++++++++++---------- test/e2e/dns_propagation_test.go | 311 +++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+), 102 deletions(-) create mode 100644 test/e2e/dns_propagation_test.go diff --git a/test/e2e/all_test.go b/test/e2e/all_test.go index f4ed961f5..128475d6f 100644 --- a/test/e2e/all_test.go +++ b/test/e2e/all_test.go @@ -19,109 +19,110 @@ func TestAll(t *testing.T) { // This call to Run() will not return until all of its // parallel subtests complete. Each "parallel" test must // invoke t.Parallel(). - t.Run("parallel", func(t *testing.T) { - t.Run("TestAWSELBConnectionIdleTimeout", TestAWSELBConnectionIdleTimeout) - t.Run("TestClientTLS", TestClientTLS) - t.Run("TestMTLSWithCRLs", TestMTLSWithCRLs) - t.Run("TestCRLUpdate", TestCRLUpdate) - t.Run("TestContainerLogging", TestContainerLogging) - t.Run("TestContainerLoggingMaxLength", TestContainerLoggingMaxLength) - t.Run("TestContainerLoggingMinLength", TestContainerLoggingMinLength) - t.Run("TestCustomErrorpages", TestCustomErrorpages) - t.Run("TestCustomIngressClass", TestCustomIngressClass) - t.Run("TestDomainNotMatchingBase", TestDomainNotMatchingBase) - t.Run("TestUnsupportedConfigOverride", TestUnsupportedConfigOverride) - t.Run("TestForwardedHeaderPolicyAppend", TestForwardedHeaderPolicyAppend) - t.Run("TestForwardedHeaderPolicyIfNone", TestForwardedHeaderPolicyIfNone) - t.Run("TestForwardedHeaderPolicyNever", TestForwardedHeaderPolicyNever) - t.Run("TestForwardedHeaderPolicyReplace", TestForwardedHeaderPolicyReplace) - t.Run("TestHAProxyTimeouts", TestHAProxyTimeouts) - t.Run("TestHAProxyTimeoutsRejection", TestHAProxyTimeoutsRejection) - t.Run("TestCookieLen", TestCookieLen) - t.Run("TestHTTPCookieCapture", TestHTTPCookieCapture) - t.Run("TestHTTPHeaderBufferSize", TestHTTPHeaderBufferSize) - t.Run("TestHTTPHeaderCapture", TestHTTPHeaderCapture) - t.Run("TestHeaderNameCaseAdjustment", TestHeaderNameCaseAdjustment) - t.Run("TestHealthCheckIntervalIngressController", TestHealthCheckIntervalIngressController) - t.Run("TestHostNetworkEndpointPublishingStrategy", TestHostNetworkEndpointPublishingStrategy) - t.Run("TestIngressControllerScale", TestIngressControllerScale) - t.Run("TestIngressControllerServiceNameCollision", TestIngressControllerServiceNameCollision) - t.Run("TestInternalLoadBalancer", TestInternalLoadBalancer) - t.Run("TestInternalLoadBalancerGlobalAccessGCP", TestInternalLoadBalancerGlobalAccessGCP) - t.Run("TestLoadBalancingAlgorithmUnsupportedConfigOverride", TestLoadBalancingAlgorithmUnsupportedConfigOverride) - t.Run("TestLocalWithFallbackOverrideForNodePortService", TestLocalWithFallbackOverrideForNodePortService) - t.Run("TestNetworkLoadBalancer", TestNetworkLoadBalancer) - t.Run("TestNodePortServiceEndpointPublishingStrategy", TestNodePortServiceEndpointPublishingStrategy) - t.Run("TestProxyProtocolAPI", TestProxyProtocolAPI) - t.Run("TestRouteAdmissionPolicy", TestRouteAdmissionPolicy) - t.Run("TestRouterCompressionParsing", TestRouterCompressionParsing) - t.Run("TestScopeChange", TestScopeChange) - t.Run("TestSyslogLogging", TestSyslogLogging) - t.Run("TestTLSSecurityProfile", TestTLSSecurityProfile) - t.Run("TestTunableMaxConnectionsInvalidValues", TestTunableMaxConnectionsInvalidValues) - t.Run("TestTunableMaxConnectionsValidValues", TestTunableMaxConnectionsValidValues) - t.Run("TestTunableRouterKubeletProbesForCustomIngressController", TestTunableRouterKubeletProbesForCustomIngressController) - t.Run("TestUniqueDomainRejection", TestUniqueDomainRejection) - t.Run("TestUniqueIdHeader", TestUniqueIdHeader) - t.Run("TestUserDefinedIngressController", TestUserDefinedIngressController) - t.Run("TestIngressOperatorCacheIsNotGlobal", TestIngressOperatorCacheIsNotGlobal) - t.Run("TestDeleteIngressControllerShouldClearRouteStatus", TestDeleteIngressControllerShouldClearRouteStatus) - t.Run("TestIngressControllerRouteSelectorUpdateShouldClearRouteStatus", TestIngressControllerRouteSelectorUpdateShouldClearRouteStatus) - t.Run("TestIngressControllerNamespaceSelectorUpdateShouldClearRouteStatus", TestIngressControllerNamespaceSelectorUpdateShouldClearRouteStatus) - t.Run("TestReloadInterval", TestReloadInterval) - t.Run("TestAWSLBTypeChange", TestAWSLBTypeChange) - t.Run("TestAllowedSourceRanges", TestAllowedSourceRanges) - t.Run("TestAllowedSourceRangesStatus", TestAllowedSourceRangesStatus) - t.Run("TestSourceRangesProgressingAndEvaluationConditionsDetectedStatuses", TestSourceRangesProgressingAndEvaluationConditionsDetectedStatuses) - t.Run("TestUnmanagedDNSToManagedDNSIngressController", TestUnmanagedDNSToManagedDNSIngressController) - t.Run("TestManagedDNSToUnmanagedDNSIngressController", TestManagedDNSToUnmanagedDNSIngressController) - t.Run("TestUnmanagedDNSToManagedDNSInternalIngressController", TestUnmanagedDNSToManagedDNSInternalIngressController) - t.Run("TestRouteMetricsControllerOnlyRouteSelector", TestRouteMetricsControllerOnlyRouteSelector) - t.Run("TestRouteMetricsControllerOnlyNamespaceSelector", TestRouteMetricsControllerOnlyNamespaceSelector) - t.Run("TestRouteMetricsControllerRouteAndNamespaceSelector", TestRouteMetricsControllerRouteAndNamespaceSelector) - t.Run("TestSetIngressControllerResponseHeaders", TestSetIngressControllerResponseHeaders) - t.Run("TestSetRouteResponseHeaders", TestSetRouteResponseHeaders) - t.Run("TestReconcileInternalService", TestReconcileInternalService) - t.Run("TestConnectTimeout", TestConnectTimeout) - t.Run("TestGatewayAPI", TestGatewayAPI) - t.Run("TestAWSLBSubnets", TestAWSLBSubnets) - t.Run("TestUnmanagedAWSLBSubnets", TestUnmanagedAWSLBSubnets) - t.Run("TestAWSEIPAllocationsForNLB", TestAWSEIPAllocationsForNLB) - t.Run("TestUnmanagedAWSEIPAllocations", TestUnmanagedAWSEIPAllocations) - }) + //t.Run("parallel", func(t *testing.T) { + // t.Run("TestAWSELBConnectionIdleTimeout", TestAWSELBConnectionIdleTimeout) + // t.Run("TestClientTLS", TestClientTLS) + // t.Run("TestMTLSWithCRLs", TestMTLSWithCRLs) + // t.Run("TestCRLUpdate", TestCRLUpdate) + // t.Run("TestContainerLogging", TestContainerLogging) + // t.Run("TestContainerLoggingMaxLength", TestContainerLoggingMaxLength) + // t.Run("TestContainerLoggingMinLength", TestContainerLoggingMinLength) + // t.Run("TestCustomErrorpages", TestCustomErrorpages) + // t.Run("TestCustomIngressClass", TestCustomIngressClass) + // t.Run("TestDomainNotMatchingBase", TestDomainNotMatchingBase) + // t.Run("TestUnsupportedConfigOverride", TestUnsupportedConfigOverride) + // t.Run("TestForwardedHeaderPolicyAppend", TestForwardedHeaderPolicyAppend) + // t.Run("TestForwardedHeaderPolicyIfNone", TestForwardedHeaderPolicyIfNone) + // t.Run("TestForwardedHeaderPolicyNever", TestForwardedHeaderPolicyNever) + // t.Run("TestForwardedHeaderPolicyReplace", TestForwardedHeaderPolicyReplace) + // t.Run("TestHAProxyTimeouts", TestHAProxyTimeouts) + // t.Run("TestHAProxyTimeoutsRejection", TestHAProxyTimeoutsRejection) + // t.Run("TestCookieLen", TestCookieLen) + // t.Run("TestHTTPCookieCapture", TestHTTPCookieCapture) + // t.Run("TestHTTPHeaderBufferSize", TestHTTPHeaderBufferSize) + // t.Run("TestHTTPHeaderCapture", TestHTTPHeaderCapture) + // t.Run("TestHeaderNameCaseAdjustment", TestHeaderNameCaseAdjustment) + // t.Run("TestHealthCheckIntervalIngressController", TestHealthCheckIntervalIngressController) + // t.Run("TestHostNetworkEndpointPublishingStrategy", TestHostNetworkEndpointPublishingStrategy) + // t.Run("TestIngressControllerScale", TestIngressControllerScale) + // t.Run("TestIngressControllerServiceNameCollision", TestIngressControllerServiceNameCollision) + // t.Run("TestInternalLoadBalancer", TestInternalLoadBalancer) + // t.Run("TestInternalLoadBalancerGlobalAccessGCP", TestInternalLoadBalancerGlobalAccessGCP) + // t.Run("TestLoadBalancingAlgorithmUnsupportedConfigOverride", TestLoadBalancingAlgorithmUnsupportedConfigOverride) + // t.Run("TestLocalWithFallbackOverrideForNodePortService", TestLocalWithFallbackOverrideForNodePortService) + // t.Run("TestNetworkLoadBalancer", TestNetworkLoadBalancer) + // t.Run("TestNodePortServiceEndpointPublishingStrategy", TestNodePortServiceEndpointPublishingStrategy) + // t.Run("TestProxyProtocolAPI", TestProxyProtocolAPI) + // t.Run("TestRouteAdmissionPolicy", TestRouteAdmissionPolicy) + // t.Run("TestRouterCompressionParsing", TestRouterCompressionParsing) + // t.Run("TestScopeChange", TestScopeChange) + // t.Run("TestSyslogLogging", TestSyslogLogging) + // t.Run("TestTLSSecurityProfile", TestTLSSecurityProfile) + // t.Run("TestTunableMaxConnectionsInvalidValues", TestTunableMaxConnectionsInvalidValues) + // t.Run("TestTunableMaxConnectionsValidValues", TestTunableMaxConnectionsValidValues) + // t.Run("TestTunableRouterKubeletProbesForCustomIngressController", TestTunableRouterKubeletProbesForCustomIngressController) + // t.Run("TestUniqueDomainRejection", TestUniqueDomainRejection) + // t.Run("TestUniqueIdHeader", TestUniqueIdHeader) + // t.Run("TestUserDefinedIngressController", TestUserDefinedIngressController) + // t.Run("TestIngressOperatorCacheIsNotGlobal", TestIngressOperatorCacheIsNotGlobal) + // t.Run("TestDeleteIngressControllerShouldClearRouteStatus", TestDeleteIngressControllerShouldClearRouteStatus) + // t.Run("TestIngressControllerRouteSelectorUpdateShouldClearRouteStatus", TestIngressControllerRouteSelectorUpdateShouldClearRouteStatus) + // t.Run("TestIngressControllerNamespaceSelectorUpdateShouldClearRouteStatus", TestIngressControllerNamespaceSelectorUpdateShouldClearRouteStatus) + // t.Run("TestReloadInterval", TestReloadInterval) + // t.Run("TestAWSLBTypeChange", TestAWSLBTypeChange) + // t.Run("TestAllowedSourceRanges", TestAllowedSourceRanges) + // t.Run("TestAllowedSourceRangesStatus", TestAllowedSourceRangesStatus) + // t.Run("TestSourceRangesProgressingAndEvaluationConditionsDetectedStatuses", TestSourceRangesProgressingAndEvaluationConditionsDetectedStatuses) + // t.Run("TestUnmanagedDNSToManagedDNSIngressController", TestUnmanagedDNSToManagedDNSIngressController) + // t.Run("TestManagedDNSToUnmanagedDNSIngressController", TestManagedDNSToUnmanagedDNSIngressController) + // t.Run("TestUnmanagedDNSToManagedDNSInternalIngressController", TestUnmanagedDNSToManagedDNSInternalIngressController) + // t.Run("TestRouteMetricsControllerOnlyRouteSelector", TestRouteMetricsControllerOnlyRouteSelector) + // t.Run("TestRouteMetricsControllerOnlyNamespaceSelector", TestRouteMetricsControllerOnlyNamespaceSelector) + // t.Run("TestRouteMetricsControllerRouteAndNamespaceSelector", TestRouteMetricsControllerRouteAndNamespaceSelector) + // t.Run("TestSetIngressControllerResponseHeaders", TestSetIngressControllerResponseHeaders) + // t.Run("TestSetRouteResponseHeaders", TestSetRouteResponseHeaders) + // t.Run("TestReconcileInternalService", TestReconcileInternalService) + // t.Run("TestConnectTimeout", TestConnectTimeout) + // t.Run("TestGatewayAPI", TestGatewayAPI) + // t.Run("TestAWSLBSubnets", TestAWSLBSubnets) + // t.Run("TestUnmanagedAWSLBSubnets", TestUnmanagedAWSLBSubnets) + // t.Run("TestAWSEIPAllocationsForNLB", TestAWSEIPAllocationsForNLB) + // t.Run("TestUnmanagedAWSEIPAllocations", TestUnmanagedAWSEIPAllocations) + //}) t.Run("serial", func(t *testing.T) { - t.Run("TestDefaultIngressControllerSteadyConditions", TestDefaultIngressControllerSteadyConditions) - t.Run("TestClusterOperatorStatusRelatedObjects", TestClusterOperatorStatusRelatedObjects) - t.Run("TestConfigurableRouteNoConsumingUserNoRBAC", TestConfigurableRouteNoConsumingUserNoRBAC) - t.Run("TestConfigurableRouteNoSecretNoRBAC", TestConfigurableRouteNoSecretNoRBAC) - t.Run("TestConfigurableRouteRBAC", TestConfigurableRouteRBAC) - t.Run("TestDefaultIngressCertificate", TestDefaultIngressCertificate) - t.Run("TestDefaultIngressClass", TestDefaultIngressClass) - t.Run("TestOperatorRecreatesItsClusterOperator", TestOperatorRecreatesItsClusterOperator) - t.Run("TestAWSLBTypeDefaulting", TestAWSLBTypeDefaulting) - t.Run("TestHstsPolicyWorks", TestHstsPolicyWorks) - t.Run("TestIngressControllerCustomEndpoints", TestIngressControllerCustomEndpoints) - t.Run("TestIngressStatus", TestIngressStatus) - t.Run("TestLocalWithFallbackOverrideForLoadBalancerService", TestLocalWithFallbackOverrideForLoadBalancerService) - t.Run("TestOperatorSteadyConditions", TestOperatorSteadyConditions) - t.Run("TestPodDisruptionBudgetExists", TestPodDisruptionBudgetExists) - t.Run("TestProxyProtocolOnAWS", TestProxyProtocolOnAWS) - t.Run("TestRouteHTTP2EnableAndDisableIngressController", TestRouteHTTP2EnableAndDisableIngressController) - t.Run("TestRouteHardStopAfterEnableOnIngressController", TestRouteHardStopAfterEnableOnIngressController) - t.Run("TestRouteHardStopAfterTestInvalidDuration", TestRouteHardStopAfterTestInvalidDuration) - t.Run("TestRouteHardStopAfterTestOneDayDuration", TestRouteHardStopAfterTestOneDayDuration) - t.Run("TestRouteHardStopAfterTestZeroLengthDuration", TestRouteHardStopAfterTestZeroLengthDuration) - t.Run("TestRouteNbthreadIngressController", TestRouteNbthreadIngressController) - t.Run("TestRouterCompressionOperation", TestRouterCompressionOperation) - t.Run("TestUpdateDefaultIngressControllerSecret", TestUpdateDefaultIngressControllerSecret) - t.Run("TestCanaryRoute", TestCanaryRoute) - t.Run("TestCanaryWithMTLS", TestCanaryWithMTLS) - t.Run("TestCanaryRouteClearsSpecHost", TestCanaryRouteClearsSpecHost) - t.Run("TestRouteHTTP2EnableAndDisableIngressConfig", TestRouteHTTP2EnableAndDisableIngressConfig) - t.Run("TestRouteHardStopAfterEnableOnIngressConfig", TestRouteHardStopAfterEnableOnIngressConfig) - t.Run("TestRouteHardStopAfterEnableOnIngressControllerHasPriorityOverIngressConfig", TestRouteHardStopAfterEnableOnIngressControllerHasPriorityOverIngressConfig) - t.Run("TestHostNetworkPortBinding", TestHostNetworkPortBinding) - t.Run("TestDashboardCreation", TestDashboardCreation) + //t.Run("TestDefaultIngressControllerSteadyConditions", TestDefaultIngressControllerSteadyConditions) + //t.Run("TestClusterOperatorStatusRelatedObjects", TestClusterOperatorStatusRelatedObjects) + //t.Run("TestConfigurableRouteNoConsumingUserNoRBAC", TestConfigurableRouteNoConsumingUserNoRBAC) + //t.Run("TestConfigurableRouteNoSecretNoRBAC", TestConfigurableRouteNoSecretNoRBAC) + //t.Run("TestConfigurableRouteRBAC", TestConfigurableRouteRBAC) + //t.Run("TestDefaultIngressCertificate", TestDefaultIngressCertificate) + //t.Run("TestDefaultIngressClass", TestDefaultIngressClass) + //t.Run("TestOperatorRecreatesItsClusterOperator", TestOperatorRecreatesItsClusterOperator) + //t.Run("TestAWSLBTypeDefaulting", TestAWSLBTypeDefaulting) + //t.Run("TestHstsPolicyWorks", TestHstsPolicyWorks) + //t.Run("TestIngressControllerCustomEndpoints", TestIngressControllerCustomEndpoints) + //t.Run("TestIngressStatus", TestIngressStatus) + //t.Run("TestLocalWithFallbackOverrideForLoadBalancerService", TestLocalWithFallbackOverrideForLoadBalancerService) + //t.Run("TestOperatorSteadyConditions", TestOperatorSteadyConditions) + //t.Run("TestPodDisruptionBudgetExists", TestPodDisruptionBudgetExists) + //t.Run("TestProxyProtocolOnAWS", TestProxyProtocolOnAWS) + //t.Run("TestRouteHTTP2EnableAndDisableIngressController", TestRouteHTTP2EnableAndDisableIngressController) + //t.Run("TestRouteHardStopAfterEnableOnIngressController", TestRouteHardStopAfterEnableOnIngressController) + //t.Run("TestRouteHardStopAfterTestInvalidDuration", TestRouteHardStopAfterTestInvalidDuration) + //t.Run("TestRouteHardStopAfterTestOneDayDuration", TestRouteHardStopAfterTestOneDayDuration) + //t.Run("TestRouteHardStopAfterTestZeroLengthDuration", TestRouteHardStopAfterTestZeroLengthDuration) + //t.Run("TestRouteNbthreadIngressController", TestRouteNbthreadIngressController) + //t.Run("TestRouterCompressionOperation", TestRouterCompressionOperation) + //t.Run("TestUpdateDefaultIngressControllerSecret", TestUpdateDefaultIngressControllerSecret) + //t.Run("TestCanaryRoute", TestCanaryRoute) + //t.Run("TestCanaryWithMTLS", TestCanaryWithMTLS) + //t.Run("TestCanaryRouteClearsSpecHost", TestCanaryRouteClearsSpecHost) + //t.Run("TestRouteHTTP2EnableAndDisableIngressConfig", TestRouteHTTP2EnableAndDisableIngressConfig) + //t.Run("TestRouteHardStopAfterEnableOnIngressConfig", TestRouteHardStopAfterEnableOnIngressConfig) + //t.Run("TestRouteHardStopAfterEnableOnIngressControllerHasPriorityOverIngressConfig", TestRouteHardStopAfterEnableOnIngressControllerHasPriorityOverIngressConfig) + //t.Run("TestHostNetworkPortBinding", TestHostNetworkPortBinding) + //t.Run("TestDashboardCreation", TestDashboardCreation) + t.Run("TestDNSPropagation", TestDNSPropagation) }) } diff --git a/test/e2e/dns_propagation_test.go b/test/e2e/dns_propagation_test.go new file mode 100644 index 000000000..33bd25a72 --- /dev/null +++ b/test/e2e/dns_propagation_test.go @@ -0,0 +1,311 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "context" + "fmt" + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "net" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sync" + "testing" + "time" +) + +func TestDNSPropagation(t *testing.T) { + if infraConfig.Status.PlatformStatus == nil { + t.Skip("test skipped on nil platform") + } + if infraConfig.Status.PlatformStatus.Type != configv1.AWSPlatformType { + t.Skipf("test skipped on platform %q", infraConfig.Status.PlatformStatus.Type) + } + + // Resolver for to resolve via 8.8.8.8 + googleResolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: time.Millisecond * time.Duration(5000), + } + return d.DialContext(ctx, network, "8.8.8.8:53") + }, + } + + // Resolver for to resolve via 10.0.0.2 + awsResolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: time.Millisecond * time.Duration(5000), + } + return d.DialContext(ctx, network, "10.0.0.2:53") + }, + } + // We need a + //client-go client in order to execute commands in the client + // pod. + kubeConfig, err := config.GetConfig() + if err != nil { + t.Fatalf("failed to get kube config: %v", err) + } + cl, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kube client: %v", err) + } + + icName := types.NamespacedName{Namespace: operatorNamespace, Name: "dns-propagate"} + domain := icName.Name + "." + dnsConfig.Spec.BaseDomain + + deployment := &appsv1.Deployment{} + deploymentName := types.NamespacedName{Namespace: controller.DefaultOperandNamespace, Name: "router-default"} + if err := kclient.Get(context.TODO(), deploymentName, deployment); err != nil { + t.Fatalf("failed to get deployment %q: %v", deploymentName, err) + } + + // We need an image that we can use for test clients. The router image + // has dig, so we can use that image. + podName := "client-local-netlookup" + image := deployment.Spec.Template.Spec.Containers[0].Image + clientPod := buildExecPod(podName, "openshift-ingress", image) + clientPodName := types.NamespacedName{ + Name: clientPod.Name, + Namespace: clientPod.Namespace, + } + if err := kclient.Create(context.TODO(), clientPod); err != nil { + t.Fatalf("failed to create pod %q: %v", clientPodName, err) + } + defer func() { + if err := kclient.Delete(context.TODO(), clientPod); err != nil { + if apierrors.IsNotFound(err) { + return + } + t.Fatalf("failed to delete pod %q: %v", clientPodName, err) + } + }() + + // Test Cases: + // Static vs. Dynamic vs. LB Address Domains + // External (Test Runner Cluster) vs. Internal (Ephemeral Cluster) Resolution + // External (Test Runner Cluster) CoreDNS vs. Google (8.8.8.8) vs. Default (AWS DNS) + + for i := 1; i <= 10; i++ { + t.Logf("------------------------------------------------") + t.Logf("Starting Test Round %d", i) + ic := newLoadBalancerController(icName, domain) + if err := waitForIngressControllerServiceDeleted(t, ic, 3*time.Minute); err != nil { + t.Fatalf("failed to wait for ingresscontroller service to be deleted: %v", err) + } + + icCreateStart := time.Now() + if err := kclient.Create(context.Background(), ic); err != nil { + t.Fatalf("expected ingresscontroller creation failed: %v", err) + } + t.Cleanup(func() { assertIngressControllerDeleted(t, kclient, ic) }) + + staticDomain := "static." + domain + + domainsToResolve := map[string]func() string{ + "STATIC": func() string { return staticDomain }, + "DYNAMIC": func() string { + // Generate 8 digit long random number + rand.Seed(time.Now().UnixNano()) + randomNumber := rand.Intn(99999999-10000000+1) + 10000000 + return fmt.Sprintf("dyn%d.%s", randomNumber, domain) + }, + } + + lbAddress := "" + var lbProvisionFinish time.Time + // Once the LB Address is set, we'll start querying that. + domainsToResolve["LB_ADDRESS"] = func() string { + // Already have it. + if lbAddress != "" { + return lbAddress + } + lbService := &corev1.Service{} + if err := kclient.Get(context.Background(), controller.LoadBalancerServiceName(ic), lbService); err != nil { + t.Logf("failed to get %q service: %v, retrying ...", controller.LoadBalancerServiceName(ic), err) + return "" + } + if len(lbService.Status.LoadBalancer.Ingress) > 0 && len(lbService.Status.LoadBalancer.Ingress[0].Hostname) > 0 { + lbAddress = lbService.Status.LoadBalancer.Ingress[0].Hostname + lbProvisionFinish = time.Now() + t.Logf("lb took %s to provision", time.Now().Sub(icCreateStart)) + } + return lbAddress + } + + // Set the number of goroutines to wait for + var wg sync.WaitGroup + + for label, getDomain := range domainsToResolve { + // CoreDNS External + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + t.Logf("[External %s @CoreDNS] starting DNS query polling", label) + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { + queryStart := time.Now() + domain := getDomain() + if domain == "" { + t.Logf("[External %s @CoreDNS] waiting for domain to populate...", label) + return false, nil + } + ips, err := net.LookupIP(domain) + if err == nil { + // t.Logf("[External %s @CoreDNS] query_time=%s total_time=%s FAILED: %v", time.Now().Sub(queryStart), time.Now().Sub(start), label, err) + //} else { + t.Logf("[External %s @CoreDNS] query_time=%s total_time=%s after_lb_provision=%s SUCCESS: resolved %s to %v", label, time.Now().Sub(queryStart), time.Now().Sub(start), time.Now().Sub(lbProvisionFinish), domain, ips) + return true, nil + } + return false, nil + }) + if err != nil { + t.Logf("[External %s @CoreDNS] failed to resolve DNS", label) + } + }() + + // Cluster DNS Resolver (Default) External + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + t.Logf("[External %s @10.0.0.2] starting DNS query polling", label) + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { + queryStart := time.Now() + domain := getDomain() + if domain == "" { + t.Logf("[External %s @10.0.0.2] waiting for domain to populate...", label) + return false, nil + } + ips, err := awsResolver.LookupHost(ctx, domain) + if err == nil { + // t.Logf("[External %s @10.0.0.2] query_time=%s total_time=%s FAILED: %v", time.Now().Sub(queryStart), time.Now().Sub(start), label, err) + //} else { + t.Logf("[External %s @10.0.0.2] query_time=%s total_time=%s after_lb_provision=%s SUCCESS: resolved %s to %v", label, time.Now().Sub(queryStart), time.Now().Sub(start), time.Now().Sub(lbProvisionFinish), domain, ips) + return true, nil + } + return false, nil + }) + if err != nil { + t.Logf("[External %s @10.0.0.2] failed to resolve DNS", label) + } + }() + + // Google External + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + t.Logf("[External %s @8.8.8.8] starting DNS query polling", label) + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { + queryStart := time.Now() + domain := getDomain() + if domain == "" { + t.Logf("[External %s @8.8.8.8] waiting for domain to populate...", label) + return false, nil + } + ips, err := googleResolver.LookupHost(ctx, domain) + if err == nil { + // t.Logf("[External %s @8.8.8.8] query_time=%s total_time=%s FAILED: %v", time.Now().Sub(queryStart), time.Now().Sub(start), label, err) + //} else { + t.Logf("[External %s @8.8.8.8] query_time=%s total_time=%s after_lb_provision=%s SUCCESS: resolved %s to %v", label, time.Now().Sub(queryStart), time.Now().Sub(start), time.Now().Sub(lbProvisionFinish), domain, ips) + return true, nil + } + return false, nil + }) + if err != nil { + t.Logf("[External %s @8.8.8.8] failed to resolve DNS", label) + } + }() + + // CoreDNS Internal + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + t.Logf("[Internal %s @CoreDNS] starting DNS query polling", label) + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { + queryStart := time.Now() + domain := getDomain() + if domain == "" { + t.Logf("[Internal %s @CoreDNS] waiting for domain to populate...", label) + return false, nil + } + stdOut, err := clusterLocalNetLookup(t, cl, kubeConfig, *clientPod, domain, "", false) + if err != nil { + t.Logf("[Internal %s @CoreDNS] error with cluster local net lookup: %v", label, err) + return true, nil + } + if stdOut != "" { + // t.Logf("[Internal %s @CoreDNS] query_time=%s total_time=%s FAILED: %v", time.Now().Sub(queryStart), time.Now().Sub(start), label, err) + //} else { + t.Logf("[Internal %s @CoreDNS] query_time=%s total_time=%s after_lb_provision=%s SUCCESS: resolved %s to %v", label, time.Now().Sub(queryStart), time.Now().Sub(start), time.Now().Sub(lbProvisionFinish), domain, stdOut) + return true, nil + } + return false, nil + }) + if err != nil { + t.Logf("[Internal %s CoreDNS] failed to resolve DNS", label) + } + }() + } + + // Wait for everything to finish + wg.Wait() + t.Logf("finished round %d...deleting ingresscontroller", i) + assertIngressControllerDeleted(t, kclient, ic) + } + t.Fatal("Failing so that we can observe logs and output!") +} + +func clusterLocalNetLookup(t *testing.T, cl *kubernetes.Clientset, kubeConfig *rest.Config, clientPod corev1.Pod, host string, resolver string, useTcp bool) (string, error) { + req := cl.CoreV1().RESTClient().Post().Resource("pods"). + Namespace(clientPod.Namespace).Name(clientPod.Name). + SubResource("exec"). + Param("container", clientPod.Spec.Containers[0].Name) + cmd := []string{ + "/bin/dig", "+short", host, + } + if resolver != "" { + cmd = append(cmd, "@"+resolver) + } + if useTcp { + cmd = append(cmd, "+tcp") + } + req.VersionedParams(&corev1.PodExecOptions{ + Command: cmd, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(kubeConfig, "POST", req.URL()) + if err != nil { + return "", err + } + var stdout, stderr bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + stdoutStr := stdout.String() + //t.Logf("command: %s\nstdout:\n%s\nstderr:\n%s\n", strings.Join(cmd, " "), stdoutStr, stderr.String()) + if err != nil { + return "", err + } + return stdoutStr, nil +}