diff --git a/Makefile b/Makefile index 0f622dbb9..d80029e5f 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,7 @@ test-unit: manifests generate fmt vet envtest ## Run unit tests. .PHONY: test-integration test-integration: ginkgo manifests generate fmt vet envtest ## Run integration tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -tags=integration -v ./test/integration + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -tags=integration -v --focus "${FOCUS}" ./test/integration .PHONY: test test: test-unit test-integration ## Run tests. diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 5879a15d6..b40166131 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -101,7 +101,7 @@ func main() { os.Exit(1) } - placement := placement.NewOCMPlacer(mgr.GetClient()) + placer := placement.NewOCMPlacer(mgr.GetClient()) provider := dnsprovider.NewProvider(mgr.GetClient()) healthMonitor := health.NewMonitor() @@ -137,7 +137,7 @@ func main() { BaseReconciler: dnsPolicyBaseReconciler, }, DNSProvider: provider.DNSProviderFactory, - Placement: placement, + Placer: placer, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DNSPolicy") os.Exit(1) @@ -179,7 +179,7 @@ func main() { if err = (&gateway.GatewayReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - Placement: placement, + Placement: placer, }).SetupWithManager(mgr, ctx); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Gateway") os.Exit(1) diff --git a/pkg/_internal/metadata/labels.go b/pkg/_internal/metadata/labels.go index a03d5c5d7..adb5ad306 100644 --- a/pkg/_internal/metadata/labels.go +++ b/pkg/_internal/metadata/labels.go @@ -15,6 +15,13 @@ func HasLabel(obj metav1.Object, key string) bool { return ok } +func GetLabel(obj metav1.Object, key string) string { + if !HasLabel(obj, key) { + return "" + } + return obj.GetLabels()[key] +} + func HasLabelsContaining(obj metav1.Object, key string) (bool, map[string]string) { matches := map[string]string{} labels := obj.GetLabels() diff --git a/pkg/apis/v1alpha1/dnshealthcheckprobe_types.go b/pkg/apis/v1alpha1/dnshealthcheckprobe_types.go index 7e9005777..8221892e2 100644 --- a/pkg/apis/v1alpha1/dnshealthcheckprobe_types.go +++ b/pkg/apis/v1alpha1/dnshealthcheckprobe_types.go @@ -51,7 +51,7 @@ type DNSHealthCheckProbeStatus struct { ConsecutiveFailures int `json:"consecutiveFailures,omitempty"` Reason string `json:"reason,omitempty"` Status int `json:"status,omitempty"` - Healthy bool `json:"healthy"` + Healthy *bool `json:"healthy"` } //+kubebuilder:object:root=true diff --git a/pkg/apis/v1alpha1/dnsrecord_types.go b/pkg/apis/v1alpha1/dnsrecord_types.go index f5c847209..3b8fe43f2 100644 --- a/pkg/apis/v1alpha1/dnsrecord_types.go +++ b/pkg/apis/v1alpha1/dnsrecord_types.go @@ -107,7 +107,7 @@ type DNSRecordSpec struct { ManagedZoneRef *ManagedZoneReference `json:"managedZone,omitempty"` // +kubebuilder:validation:MinItems=1 // +optional - Endpoints []*Endpoint `json:"endpoints"` + Endpoints []*Endpoint `json:"endpoints,omitempty"` } // DNSRecordStatus defines the observed state of DNSRecord diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 1f00e1b8c..2aae0a414 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -232,6 +232,11 @@ func (in *DNSHealthCheckProbeSpec) DeepCopy() *DNSHealthCheckProbeSpec { func (in *DNSHealthCheckProbeStatus) DeepCopyInto(out *DNSHealthCheckProbeStatus) { *out = *in in.LastCheckedAt.DeepCopyInto(&out.LastCheckedAt) + if in.Healthy != nil { + in, out := &in.Healthy, &out.Healthy + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSHealthCheckProbeStatus. diff --git a/pkg/controllers/dnshealthcheckprobe/dnshealthcheckprobe_controller.go b/pkg/controllers/dnshealthcheckprobe/dnshealthcheckprobe_controller.go index b29a613dd..f149d8e19 100644 --- a/pkg/controllers/dnshealthcheckprobe/dnshealthcheckprobe_controller.go +++ b/pkg/controllers/dnshealthcheckprobe/dnshealthcheckprobe_controller.go @@ -159,14 +159,12 @@ func getAdditionalHeaders(ctx context.Context, clt client.Client, probeObj *v1al return additionalHeaders, fmt.Errorf("error retrieving additional headers secret %v/%v: %w", secretKey.Namespace, secretKey.Name, err) } else if err != nil { probeError := fmt.Errorf("error retrieving additional headers secret %v/%v: %w", secretKey.Namespace, secretKey.Name, err) - probeObj.Status.Healthy = false probeObj.Status.ConsecutiveFailures = 0 probeObj.Status.Reason = "additional headers secret not found" return additionalHeaders, probeError } for k, v := range additionalHeadersSecret.Data { if strings.ContainsAny(strings.TrimSpace(k), " \t") { - probeObj.Status.Healthy = false probeObj.Status.ConsecutiveFailures = 0 probeObj.Status.Reason = "invalid header found: " + k return nil, fmt.Errorf("invalid header, must not contain whitespace '%v': %w", k, ErrInvalidHeader) diff --git a/pkg/controllers/dnshealthcheckprobe/notifier.go b/pkg/controllers/dnshealthcheckprobe/notifier.go index dd69520ac..cb3da645b 100644 --- a/pkg/controllers/dnshealthcheckprobe/notifier.go +++ b/pkg/controllers/dnshealthcheckprobe/notifier.go @@ -3,6 +3,8 @@ package dnshealthcheckprobe import ( "context" + "github.com/aws/aws-sdk-go/aws" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,7 +35,11 @@ func (n StatusUpdateProbeNotifier) Notify(ctx context.Context, result health.Pro // Increase the number of consecutive failures if it failed previously if !result.Healthy { - if probeObj.Status.Healthy { + probeHealthy := true + if probeObj.Status.Healthy != nil { + probeHealthy = *probeObj.Status.Healthy + } + if probeHealthy { probeObj.Status.ConsecutiveFailures = 1 } else { probeObj.Status.ConsecutiveFailures++ @@ -43,7 +49,10 @@ func (n StatusUpdateProbeNotifier) Notify(ctx context.Context, result health.Pro } probeObj.Status.LastCheckedAt = metav1.NewTime(result.CheckedAt) - probeObj.Status.Healthy = result.Healthy + if probeObj.Status.Healthy == nil { + probeObj.Status.Healthy = aws.Bool(true) + } + probeObj.Status.Healthy = &result.Healthy probeObj.Status.Reason = result.Reason probeObj.Status.Status = result.Status diff --git a/pkg/controllers/dnspolicy/dns_helper.go b/pkg/controllers/dnspolicy/dns_helper.go index c0fd102b9..86d288e49 100644 --- a/pkg/controllers/dnspolicy/dns_helper.go +++ b/pkg/controllers/dnspolicy/dns_helper.go @@ -3,6 +3,7 @@ package dnspolicy import ( "context" "fmt" + "regexp" "sort" "strconv" "strings" @@ -12,11 +13,14 @@ import ( "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "github.com/kuadrant/kuadrant-operator/pkg/common" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/_internal/slice" "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" @@ -117,13 +121,13 @@ func (dh *dnsHelper) getDNSRecordForListener(ctx context.Context, listener gatew return dnsRecord, nil } -func withDNSRecord[T metav1.Object](dnsRecord *v1alpha1.DNSRecord, obj T) T { +func withGatewayListener[T metav1.Object](gateway common.GatewayWrapper, listener gatewayv1beta1.Listener, obj T) T { if obj.GetAnnotations() == nil { obj.SetAnnotations(map[string]string{}) } - obj.GetAnnotations()["dnsrecord-name"] = dnsRecord.Name - obj.GetAnnotations()["dnsrecord-namespace"] = dnsRecord.Namespace + obj.GetAnnotations()["dnsrecord-name"] = fmt.Sprintf("%s-%s", gateway.Name, listener.Name) + obj.GetAnnotations()["dnsrecord-namespace"] = gateway.Namespace return obj } @@ -167,7 +171,7 @@ func withDNSRecord[T metav1.Object](dnsRecord *v1alpha1.DNSRecord, obj T) T { // ab2.lb-a1b2.shop.example.com A 192.22.2.3 // ab3.lb-a1b2.shop.example.com A 192.22.2.4 -func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClusterGatewayTarget, dnsRecord *v1alpha1.DNSRecord, listener gatewayv1beta1.Listener) error { +func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClusterGatewayTarget, dnsRecord *v1alpha1.DNSRecord, dnsPolicy *v1alpha1.DNSPolicy, listener gatewayv1beta1.Listener) error { old := dnsRecord.DeepCopy() gwListenerHost := string(*listener.Hostname) @@ -235,6 +239,7 @@ func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClust } endpoint.SetProviderSpecific(dns.ProviderSpecificGeoCode, string(geoCode)) + newEndpoints = append(newEndpoints, endpoint) } @@ -250,14 +255,92 @@ func (dh *dnsHelper) setEndpoints(ctx context.Context, mcgTarget *dns.MultiClust sort.Slice(newEndpoints, func(i, j int) bool { return newEndpoints[i].SetID() < newEndpoints[j].SetID() }) - dnsRecord.Spec.Endpoints = newEndpoints + probes, err := dh.getDNSHealthCheckProbes(ctx, mcgTarget.Gateway, dnsPolicy) + if err != nil { + return err + } + + // if the checks on endpoints based on probes results in there being no healthy endpoints + // ready to publish we'll publish the full set so storing those + var storeEndpoints []*v1alpha1.Endpoint + storeEndpoints = append(storeEndpoints, newEndpoints...) + // count will track whether a new endpoint has been removed. + // first newEndpoints are checked based on probe status and removed if unhealthy true and the consecutive failures are greater than the threshold. + removedEndpoints := 0 + for i := 0; i < len(newEndpoints); i++ { + checkProbes := getProbesForEndpoint(newEndpoints[i], probes) + if len(checkProbes) == 0 { + continue + } + for _, probe := range checkProbes { + probeHealthy := true + if probe.Status.Healthy != nil { + probeHealthy = *probe.Status.Healthy + } + // if any probe for any target is reporting unhealthy remove it from the endpoint list + if !probeHealthy && probe.Spec.FailureThreshold != nil && probe.Status.ConsecutiveFailures >= *probe.Spec.FailureThreshold { + newEndpoints = append(newEndpoints[:i], newEndpoints[i+1:]...) + removedEndpoints++ + i-- + break + } + } + } + // after checkProbes are checked the newEndpoints is looped through until count is 0 + // if any are found that need to be removed because a parent with no children present + // the count will be incremented so that the newEndpoints will be traversed again such that only when a loop occurs where no + // endpoints have been removed can we consider the endpoint list to be cleaned + ipPattern := `\b(?:\d{1,3}\.){3}\d{1,3}\b` + re := regexp.MustCompile(ipPattern) + + for removedEndpoints > 0 { + endpointsLoop: + for i := 0; i < len(newEndpoints); i++ { + checkEndpoint := newEndpoints[i] + for _, target := range checkEndpoint.Targets { + if len(re.FindAllString(target, -1)) > 0 { + // don't check the children of targets which are ips. + continue endpointsLoop + } + } + children := getNumChildrenOfParent(newEndpoints, newEndpoints[i]) + if children == 0 { + newEndpoints = append(newEndpoints[:i], newEndpoints[i+1:]...) + removedEndpoints++ + } + } + removedEndpoints-- + } + + // if there are no healthy endpoints after checking, publish the full set before checks + if len(newEndpoints) == 0 { + dnsRecord.Spec.Endpoints = storeEndpoints + } else { + dnsRecord.Spec.Endpoints = newEndpoints + } if !equality.Semantic.DeepEqual(old, dnsRecord) { return dh.Update(ctx, dnsRecord) } return nil } +func getNumChildrenOfParent(endpoints []*v1alpha1.Endpoint, parent *v1alpha1.Endpoint) int { + return len(findChildren(endpoints, parent)) +} + +func findChildren(endpoints []*v1alpha1.Endpoint, parent *v1alpha1.Endpoint) []*v1alpha1.Endpoint { + var foundEPs []*v1alpha1.Endpoint + for _, endpoint := range endpoints { + for _, target := range parent.Targets { + if target == endpoint.DNSName { + foundEPs = append(foundEPs, endpoint) + } + } + } + return foundEPs +} + func createOrUpdateEndpoint(dnsName string, targets v1alpha1.Targets, recordType v1alpha1.DNSRecordType, setIdentifier string, recordTTL v1alpha1.TTL, currentEndpoints map[string]*v1alpha1.Endpoint) (endpoint *v1alpha1.Endpoint) { ok := false @@ -355,3 +438,29 @@ func (r *dnsHelper) deleteDNSRecordForListener(ctx context.Context, owner metav1 func isWildCardListener(l gatewayv1beta1.Listener) bool { return strings.HasPrefix(string(*l.Hostname), "*") } + +func (dh *dnsHelper) getDNSHealthCheckProbes(ctx context.Context, gateway *gatewayv1beta1.Gateway, dnsPolicy *v1alpha1.DNSPolicy) ([]*v1alpha1.DNSHealthCheckProbe, error) { + list := &v1alpha1.DNSHealthCheckProbeList{} + if err := dh.List(ctx, list, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(commonDNSRecordLabels(client.ObjectKeyFromObject(gateway), client.ObjectKeyFromObject(dnsPolicy))), + Namespace: dnsPolicy.Namespace, + }); err != nil { + return nil, err + } + + return slice.MapErr(list.Items, func(obj v1alpha1.DNSHealthCheckProbe) (*v1alpha1.DNSHealthCheckProbe, error) { + return &obj, nil + }) +} + +func getProbesForEndpoint(endpoint *v1alpha1.Endpoint, probes []*v1alpha1.DNSHealthCheckProbe) []*v1alpha1.DNSHealthCheckProbe { + retProbes := []*v1alpha1.DNSHealthCheckProbe{} + for _, probe := range probes { + for _, target := range endpoint.Targets { + if strings.Contains(probe.Name, target) { + retProbes = append(retProbes, probe) + } + } + } + return retProbes +} diff --git a/pkg/controllers/dnspolicy/dns_helper_test.go b/pkg/controllers/dnspolicy/dns_helper_test.go index a49d51fa1..bfdf22496 100644 --- a/pkg/controllers/dnspolicy/dns_helper_test.go +++ b/pkg/controllers/dnspolicy/dns_helper_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" + "github.com/aws/aws-sdk-go/aws" + "k8s.io/apimachinery/pkg/api/equality" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -409,22 +411,19 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { mcgTarget *dns.MultiClusterGatewayTarget listener gatewayv1beta1.Listener dnsRecord *v1alpha1.DNSRecord + dnsPolicy *v1alpha1.DNSPolicy + probeOne *v1alpha1.DNSHealthCheckProbe + probeTwo *v1alpha1.DNSHealthCheckProbe wantSpec *v1alpha1.DNSRecordSpec wantErr bool }{ { - name: "test wildcard listener wieghted", + name: "test wildcard listener weighted", listener: getTestListener("*.example.com"), mcgTarget: &dns.MultiClusterGatewayTarget{ - Gateway: &gatewayv1beta1.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "testgw", - Namespace: "testns", - }, - }, + Gateway: &gatewayv1beta1.Gateway{}, ClusterGatewayTargets: []dns.ClusterGatewayTarget{ { - ClusterGateway: &dns.ClusterGateway{ Cluster: &testutil.TestResource{ ObjectMeta: v1.ObjectMeta{ @@ -470,19 +469,32 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { Name: "test.example.com", }, }, + dnsPolicy: &v1alpha1.DNSPolicy{}, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testOne", + Namespace: "namespace", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testTwo", + Namespace: "namespace", + }, + }, wantSpec: &v1alpha1.DNSRecordSpec{ Endpoints: []*v1alpha1.Endpoint{ { - DNSName: "20qri0.lb-0ecjaw.example.com", + DNSName: "20qri0.lb-oe3k96.example.com", Targets: []string{"1.1.1.1", "2.2.2.2"}, RecordType: "A", RecordTTL: dns.DefaultTTL, }, { - DNSName: "default.lb-0ecjaw.example.com", - Targets: []string{"20qri0.lb-0ecjaw.example.com"}, + DNSName: "default.lb-oe3k96.example.com", + Targets: []string{"20qri0.lb-oe3k96.example.com"}, RecordType: "CNAME", - SetIdentifier: "20qri0.lb-0ecjaw.example.com", + SetIdentifier: "20qri0.lb-oe3k96.example.com", RecordTTL: dns.DefaultTTL, ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ { @@ -492,7 +504,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "default.lb-0ecjaw.example.com", + DNSName: "default.lb-oe3k96.example.com", Targets: []string{"mylb.example.com"}, RecordType: "CNAME", SetIdentifier: "mylb.example.com", @@ -505,8 +517,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.example.com", - Targets: []string{"default.lb-0ecjaw.example.com"}, + DNSName: "lb-oe3k96.example.com", + Targets: []string{"default.lb-oe3k96.example.com"}, RecordType: "CNAME", SetIdentifier: "default", RecordTTL: dns.DefaultCnameTTL, @@ -519,7 +531,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, { DNSName: "*.example.com", - Targets: []string{"lb-0ecjaw.example.com"}, + Targets: []string{"lb-oe3k96.example.com"}, RecordType: "CNAME", RecordTTL: dns.DefaultCnameTTL, }, @@ -530,12 +542,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { name: "sets geo weighted endpoints wildcard", listener: getTestListener("*.example.com"), mcgTarget: &dns.MultiClusterGatewayTarget{ - Gateway: &gatewayv1beta1.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "testgw", - Namespace: "testns", - }, - }, + Gateway: &gatewayv1beta1.Gateway{}, ClusterGatewayTargets: []dns.ClusterGatewayTarget{ { @@ -589,19 +596,32 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { Name: "gw-test", }, }, + dnsPolicy: &v1alpha1.DNSPolicy{}, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testOne", + Namespace: "namespace", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testTwo", + Namespace: "namespace", + }, + }, wantSpec: &v1alpha1.DNSRecordSpec{ Endpoints: []*v1alpha1.Endpoint{ { - DNSName: "20qri0.lb-0ecjaw.example.com", + DNSName: "20qri0.lb-oe3k96.example.com", Targets: []string{"1.1.1.1", "2.2.2.2"}, RecordType: "A", RecordTTL: dns.DefaultTTL, }, { - DNSName: "na.lb-0ecjaw.example.com", - Targets: []string{"20qri0.lb-0ecjaw.example.com"}, + DNSName: "na.lb-oe3k96.example.com", + Targets: []string{"20qri0.lb-oe3k96.example.com"}, RecordType: "CNAME", - SetIdentifier: "20qri0.lb-0ecjaw.example.com", + SetIdentifier: "20qri0.lb-oe3k96.example.com", RecordTTL: dns.DefaultTTL, ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ { @@ -611,7 +631,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "ie.lb-0ecjaw.example.com", + DNSName: "ie.lb-oe3k96.example.com", Targets: []string{"mylb.example.com"}, RecordType: "CNAME", SetIdentifier: "mylb.example.com", @@ -624,8 +644,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.example.com", - Targets: []string{"na.lb-0ecjaw.example.com"}, + DNSName: "lb-oe3k96.example.com", + Targets: []string{"na.lb-oe3k96.example.com"}, RecordType: "CNAME", SetIdentifier: "default", RecordTTL: dns.DefaultCnameTTL, @@ -637,8 +657,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.example.com", - Targets: []string{"na.lb-0ecjaw.example.com"}, + DNSName: "lb-oe3k96.example.com", + Targets: []string{"na.lb-oe3k96.example.com"}, RecordType: "CNAME", SetIdentifier: "NA", RecordTTL: dns.DefaultCnameTTL, @@ -650,8 +670,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.example.com", - Targets: []string{"ie.lb-0ecjaw.example.com"}, + DNSName: "lb-oe3k96.example.com", + Targets: []string{"ie.lb-oe3k96.example.com"}, RecordType: "CNAME", SetIdentifier: "IE", RecordTTL: dns.DefaultCnameTTL, @@ -664,7 +684,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, { DNSName: "*.example.com", - Targets: []string{"lb-0ecjaw.example.com"}, + Targets: []string{"lb-oe3k96.example.com"}, RecordType: "CNAME", RecordTTL: dns.DefaultCnameTTL, }, @@ -729,6 +749,19 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { Name: "test.example.com", }, }, + dnsPolicy: &v1alpha1.DNSPolicy{}, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testOne", + Namespace: "namespace", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testTwo", + Namespace: "namespace", + }, + }, wantSpec: &v1alpha1.DNSRecordSpec{ Endpoints: []*v1alpha1.Endpoint{ { @@ -789,12 +822,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { name: "sets geo weighted endpoints", listener: getTestListener("test.example.com"), mcgTarget: &dns.MultiClusterGatewayTarget{ - Gateway: &gatewayv1beta1.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "testgw", - Namespace: "testns", - }, - }, + Gateway: &gatewayv1beta1.Gateway{}, ClusterGatewayTargets: []dns.ClusterGatewayTarget{ { @@ -848,19 +876,32 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { Name: "test.example.com", }, }, + dnsPolicy: &v1alpha1.DNSPolicy{}, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testOne", + Namespace: "namespace", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testTwo", + Namespace: "namespace", + }, + }, wantSpec: &v1alpha1.DNSRecordSpec{ Endpoints: []*v1alpha1.Endpoint{ { - DNSName: "20qri0.lb-0ecjaw.test.example.com", + DNSName: "20qri0.lb-oe3k96.test.example.com", Targets: []string{"1.1.1.1", "2.2.2.2"}, RecordType: "A", RecordTTL: dns.DefaultTTL, }, { - DNSName: "na.lb-0ecjaw.test.example.com", - Targets: []string{"20qri0.lb-0ecjaw.test.example.com"}, + DNSName: "na.lb-oe3k96.test.example.com", + Targets: []string{"20qri0.lb-oe3k96.test.example.com"}, RecordType: "CNAME", - SetIdentifier: "20qri0.lb-0ecjaw.test.example.com", + SetIdentifier: "20qri0.lb-oe3k96.test.example.com", RecordTTL: dns.DefaultTTL, ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ { @@ -870,7 +911,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "ie.lb-0ecjaw.test.example.com", + DNSName: "ie.lb-oe3k96.test.example.com", Targets: []string{"mylb.example.com"}, RecordType: "CNAME", SetIdentifier: "mylb.example.com", @@ -883,8 +924,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.test.example.com", - Targets: []string{"na.lb-0ecjaw.test.example.com"}, + DNSName: "lb-oe3k96.test.example.com", + Targets: []string{"na.lb-oe3k96.test.example.com"}, RecordType: "CNAME", SetIdentifier: "default", RecordTTL: dns.DefaultCnameTTL, @@ -896,8 +937,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.test.example.com", - Targets: []string{"na.lb-0ecjaw.test.example.com"}, + DNSName: "lb-oe3k96.test.example.com", + Targets: []string{"na.lb-oe3k96.test.example.com"}, RecordType: "CNAME", SetIdentifier: "NA", RecordTTL: dns.DefaultCnameTTL, @@ -909,8 +950,8 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, }, { - DNSName: "lb-0ecjaw.test.example.com", - Targets: []string{"ie.lb-0ecjaw.test.example.com"}, + DNSName: "lb-oe3k96.test.example.com", + Targets: []string{"ie.lb-oe3k96.test.example.com"}, RecordType: "CNAME", SetIdentifier: "IE", RecordTTL: dns.DefaultCnameTTL, @@ -923,7 +964,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { }, { DNSName: "test.example.com", - Targets: []string{"lb-0ecjaw.test.example.com"}, + Targets: []string{"lb-oe3k96.test.example.com"}, RecordType: "CNAME", RecordTTL: dns.DefaultCnameTTL, }, @@ -934,12 +975,7 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { name: "sets no endpoints when no target addresses", listener: getTestListener("test.example.com"), mcgTarget: &dns.MultiClusterGatewayTarget{ - Gateway: &gatewayv1beta1.Gateway{ - ObjectMeta: v1.ObjectMeta{ - Name: "testgw", - Namespace: "testns", - }, - }, + Gateway: &gatewayv1beta1.Gateway{}, ClusterGatewayTargets: []dns.ClusterGatewayTarget{ { @@ -979,16 +1015,644 @@ func Test_dnsHelper_setEndpoints(t *testing.T) { Name: "test.example.com", }, }, + dnsPolicy: &v1alpha1.DNSPolicy{}, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testOne", + Namespace: "namespace", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Name: "testTwo", + Namespace: "namespace", + }, + }, wantSpec: &v1alpha1.DNSRecordSpec{ Endpoints: []*v1alpha1.Endpoint{}, }, }, + { + name: "test endpoint presence when probe is present but no health status is reported yet", + listener: getTestListener("*.example.com"), + mcgTarget: &dns.MultiClusterGatewayTarget{ + Gateway: &gatewayv1beta1.Gateway{ + ObjectMeta: v1.ObjectMeta{ + Name: "testgw", + Namespace: "testns", + }, + }, + ClusterGatewayTargets: []dns.ClusterGatewayTarget{ + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-1", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "1.1.1.1", + }, + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "2.2.2.2", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-2", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.HostnameAddressType), + Value: "mylb.example.com", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + }, + }, + dnsRecord: &v1alpha1.DNSRecord{ + ObjectMeta: v1.ObjectMeta{ + Name: "test.example.com", + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: v1.ObjectMeta{ + Name: "testpolicy", + Namespace: "testns", + }, + }, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "1.1.1.1-test.example.com", + Namespace: "testns", + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "2.2.2.2-test.example.com", + Namespace: "testns", + }, + }, + wantSpec: &v1alpha1.DNSRecordSpec{ + Endpoints: []*v1alpha1.Endpoint{ + { + DNSName: "20qri0.lb-0ecjaw.example.com", + Targets: []string{"1.1.1.1", "2.2.2.2"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "default.lb-0ecjaw.example.com", + Targets: []string{"20qri0.lb-0ecjaw.example.com"}, + RecordType: "CNAME", + SetIdentifier: "20qri0.lb-0ecjaw.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "default.lb-0ecjaw.example.com", + Targets: []string{"mylb.example.com"}, + RecordType: "CNAME", + SetIdentifier: "mylb.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-0ecjaw.example.com", + Targets: []string{"default.lb-0ecjaw.example.com"}, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: dns.DefaultCnameTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "*.example.com", + Targets: []string{"lb-0ecjaw.example.com"}, + RecordType: "CNAME", + RecordTTL: dns.DefaultCnameTTL, + }, + }, + }, + }, + { + name: "test endpoint presence when probe is present but healthy status is reported", + listener: getTestListener("test.example.com"), + mcgTarget: &dns.MultiClusterGatewayTarget{ + Gateway: &gatewayv1beta1.Gateway{ + ObjectMeta: v1.ObjectMeta{ + Name: "testgw", + Namespace: "testns", + }, + }, + ClusterGatewayTargets: []dns.ClusterGatewayTarget{ + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-1", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "1.1.1.1", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-1", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "2.2.2.2", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + }, + }, + dnsRecord: &v1alpha1.DNSRecord{ + ObjectMeta: v1.ObjectMeta{ + Name: "test.example.com", + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: v1.ObjectMeta{ + Name: "testpolicy", + Namespace: "testns", + }, + }, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "1.1.1.1-test.example.com", + Namespace: "testns", + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(true), + }, + }, + + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "2.2.2.2-test.example.com", + Namespace: "testns", + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(true), + }, + }, + wantSpec: &v1alpha1.DNSRecordSpec{ + Endpoints: []*v1alpha1.Endpoint{ + { + DNSName: "20qri0.lb-0ecjaw.test.example.com", + Targets: []string{"1.1.1.1"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "20qri0.lb-0ecjaw.test.example.com", + Targets: []string{"2.2.2.2"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "default.lb-0ecjaw.test.example.com", + Targets: []string{"20qri0.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "20qri0.lb-0ecjaw.test.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "default.lb-0ecjaw.test.example.com", + Targets: []string{"20qri0.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "20qri0.lb-0ecjaw.test.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-0ecjaw.test.example.com", + Targets: []string{"default.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: dns.DefaultCnameTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "test.example.com", + Targets: []string{"lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + RecordTTL: dns.DefaultCnameTTL, + }, + }, + }, + }, + { + name: "test removal of endpoint based on probe", + listener: getTestListener("test.example.com"), + mcgTarget: &dns.MultiClusterGatewayTarget{ + Gateway: &gatewayv1beta1.Gateway{ + ObjectMeta: v1.ObjectMeta{ + Name: "testgw", + Namespace: "testns", + }, + }, + ClusterGatewayTargets: []dns.ClusterGatewayTarget{ + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-1", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "2.2.2.2", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-2", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "1.1.1.1", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + }, + }, + dnsRecord: &v1alpha1.DNSRecord{ + ObjectMeta: v1.ObjectMeta{ + Name: "test.example.com", + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: v1.ObjectMeta{ + Name: "testpolicy", + Namespace: "testns", + }, + }, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "1.1.1.1-test.example.com", + Namespace: "testns", + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + FailureThreshold: aws.Int(4), + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(false), + ConsecutiveFailures: 54, + }, + }, + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "2.2.2.2-test.example.com", + Namespace: "testns", + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + FailureThreshold: aws.Int(4), + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(false), + ConsecutiveFailures: 2, + }, + }, + wantSpec: &v1alpha1.DNSRecordSpec{ + Endpoints: []*v1alpha1.Endpoint{ + { + DNSName: "20qri0.lb-0ecjaw.test.example.com", + Targets: []string{"2.2.2.2"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "default.lb-0ecjaw.test.example.com", + Targets: []string{"20qri0.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "20qri0.lb-0ecjaw.test.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-0ecjaw.test.example.com", + Targets: []string{"default.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: dns.DefaultCnameTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "test.example.com", + Targets: []string{"lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + RecordTTL: dns.DefaultCnameTTL, + }, + }, + }, + }, + { + name: "test no removal of endpoint when all probes are unhealthy", + listener: getTestListener("test.example.com"), + mcgTarget: &dns.MultiClusterGatewayTarget{ + Gateway: &gatewayv1beta1.Gateway{ + ObjectMeta: v1.ObjectMeta{ + Name: "testgw", + Namespace: "testns", + }, + }, + ClusterGatewayTargets: []dns.ClusterGatewayTarget{ + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-1", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "1.1.1.1", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + { + + ClusterGateway: &dns.ClusterGateway{ + Cluster: &testutil.TestResource{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-cluster-2", + }, + }, + GatewayAddresses: []gatewayv1beta1.GatewayAddress{ + { + Type: testutil.Pointer(gatewayv1beta1.IPAddressType), + Value: "2.2.2.2", + }, + }, + }, + Geo: testutil.Pointer(dns.GeoCode("default")), + Weight: testutil.Pointer(120), + }, + }, + }, + dnsRecord: &v1alpha1.DNSRecord{ + ObjectMeta: v1.ObjectMeta{ + Name: "test.example.com", + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: v1.ObjectMeta{ + Name: "testpolicy", + Namespace: "testns", + }, + }, + probeOne: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "1.1.1.1-test.example.com", + Namespace: "testns", + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + FailureThreshold: aws.Int(4), + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(false), + ConsecutiveFailures: 6, + }, + }, + + probeTwo: &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: v1.ObjectMeta{ + Labels: commonDNSRecordLabels( + types.NamespacedName{ + Namespace: "testns", + Name: "testgw"}, + types.NamespacedName{ + Namespace: "testns", + Name: "testpolicy", + }), + Name: "2.2.2.2-test.example.com", + Namespace: "testns", + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + FailureThreshold: aws.Int(4), + }, + Status: v1alpha1.DNSHealthCheckProbeStatus{ + Healthy: aws.Bool(false), + ConsecutiveFailures: 6, + }, + }, + wantSpec: &v1alpha1.DNSRecordSpec{ + Endpoints: []*v1alpha1.Endpoint{ + { + DNSName: "20qri0.lb-0ecjaw.test.example.com", + Targets: []string{"1.1.1.1"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "2pj3we.lb-0ecjaw.test.example.com", + Targets: []string{"2.2.2.2"}, + RecordType: "A", + RecordTTL: dns.DefaultTTL, + }, + { + DNSName: "default.lb-0ecjaw.test.example.com", + Targets: []string{"20qri0.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "20qri0.lb-0ecjaw.test.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "default.lb-0ecjaw.test.example.com", + Targets: []string{"2pj3we.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "2pj3we.lb-0ecjaw.test.example.com", + RecordTTL: dns.DefaultTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-0ecjaw.test.example.com", + Targets: []string{"default.lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: dns.DefaultCnameTTL, + ProviderSpecific: []v1alpha1.ProviderSpecificProperty{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "test.example.com", + Targets: []string{"lb-0ecjaw.test.example.com"}, + RecordType: "CNAME", + RecordTTL: dns.DefaultCnameTTL, + }, + }, + }, + }, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - f := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(testCase.dnsRecord).Build() + f := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(testCase.dnsRecord, testCase.probeOne, testCase.probeTwo).Build() s := dnsHelper{Client: f} - if err := s.setEndpoints(context.TODO(), testCase.mcgTarget, testCase.dnsRecord, testCase.listener); (err != nil) != testCase.wantErr { + if err := s.setEndpoints(context.TODO(), testCase.mcgTarget, testCase.dnsRecord, testCase.dnsPolicy, testCase.listener); (err != nil) != testCase.wantErr { t.Errorf("SetEndpoints() error = %v, wantErr %v", err, testCase.wantErr) } diff --git a/pkg/controllers/dnspolicy/dnspolicy_controller.go b/pkg/controllers/dnspolicy/dnspolicy_controller.go index 63d7868ea..5dce06243 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_controller.go +++ b/pkg/controllers/dnspolicy/dnspolicy_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "reflect" + "github.com/kuadrant/authorino/pkg/log" clusterv1 "open-cluster-management.io/api/cluster/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -63,7 +64,7 @@ type DNSPolicyReconciler struct { reconcilers.TargetRefReconciler DNSProvider dns.DNSProviderFactory dnsHelper dnsHelper - Placement gateway.GatewayPlacer + Placer gateway.GatewayPlacer } //+kubebuilder:rbac:groups=kuadrant.io,resources=dnspolicies,verbs=get;list;watch;create;update;patch;delete @@ -78,6 +79,7 @@ func (r *DNSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( previous := &v1alpha1.DNSPolicy{} if err := r.Client().Get(ctx, req.NamespacedName, previous); err != nil { + log.Info("error getting dns policy", "error", err) return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -205,6 +207,7 @@ func (r *DNSPolicyReconciler) deleteResources(ctx context.Context, dnsPolicy *v1 } if err := r.reconcileDNSRecords(ctx, dnsPolicy, gatewayDiffObj); err != nil { + log.V(3).Info("error reconciling DNS records from delete, returning", "error", err) return err } @@ -290,6 +293,7 @@ func (r *DNSPolicyReconciler) updateGatewayCondition(ctx context.Context, condit func (r *DNSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { gatewayEventMapper := events.NewGatewayEventMapper(r.Logger(), &DNSPolicyRefsConfig{}, "dnspolicy") clusterEventMapper := events.NewClusterEventMapper(r.Logger(), r.Client(), &DNSPolicyRefsConfig{}, "dnspolicy") + probeEventMapper := events.NewProbeEventMapper(r.Logger(), DNSPolicyBackRefAnnotation, "dnspolicy") r.dnsHelper = dnsHelper{Client: r.Client()} return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.DNSPolicy{}). @@ -301,5 +305,9 @@ func (r *DNSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { &source.Kind{Type: &clusterv1.ManagedCluster{}}, handler.EnqueueRequestsFromMapFunc(clusterEventMapper.MapToPolicy), ). + Watches( + &source.Kind{Type: &v1alpha1.DNSHealthCheckProbe{}}, + handler.EnqueueRequestsFromMapFunc(probeEventMapper.MapToPolicy), + ). Complete(r) } diff --git a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go index 71f914de3..7ffc94463 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go +++ b/pkg/controllers/dnspolicy/dnspolicy_dnsrecords.go @@ -43,11 +43,13 @@ func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, ga log := crlog.FromContext(ctx) if err := r.dnsHelper.removeDNSForDeletedListeners(ctx, gateway); err != nil { + log.V(3).Info("error removing DNS for deleted listeners") return err } - placed, err := r.Placement.GetPlacedClusters(ctx, gateway) + placed, err := r.Placer.GetPlacedClusters(ctx, gateway) if err != nil { + log.V(3).Info("error getting placed clusters") return err } clusters := placed.UnsortedList() @@ -68,18 +70,18 @@ func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, ga for _, downstreamCluster := range clusters { // Only consider host for dns if there's at least 1 attached route to the listener for this host in *any* gateway - log.V(3).Info("checking downstream", "listener ", listener.Name) - attached, err := r.Placement.ListenerTotalAttachedRoutes(ctx, gateway, string(listener.Name), downstreamCluster) + log.V(1).Info("checking downstream", "listener ", listener.Name) + attached, err := r.Placer.ListenerTotalAttachedRoutes(ctx, gateway, string(listener.Name), downstreamCluster) if err != nil { log.Error(err, "failed to get total attached routes for listener ", "listener", listener.Name) continue } if attached == 0 { - log.V(3).Info("no attached routes for ", "listener", listener.Name, "cluster ", downstreamCluster) + log.V(1).Info("no attached routes for ", "listener", listener.Name, "cluster ", downstreamCluster) continue } - log.V(3).Info("hostHasAttachedRoutes", "host", listener.Name, "hostHasAttachedRoutes", attached) - cg, err := r.Placement.GetClusterGateway(ctx, gateway, downstreamCluster) + log.V(1).Info("hostHasAttachedRoutes", "host", listener.Name, "hostHasAttachedRoutes", attached) + cg, err := r.Placer.GetClusterGateway(ctx, gateway, downstreamCluster) if err != nil { return fmt.Errorf("get cluster gateway failed: %s", err) } @@ -111,7 +113,7 @@ func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, ga } log.Info("setting dns dnsTargets for gateway listener", "listener", dnsRecord.Name, "values", mcgTarget) - if err := r.dnsHelper.setEndpoints(ctx, mcgTarget, dnsRecord, listener); err != nil { + if err := r.dnsHelper.setEndpoints(ctx, mcgTarget, dnsRecord, dnsPolicy, listener); err != nil { return fmt.Errorf("failed to add dns record dnsTargets %s %v", err, mcgTarget) } } diff --git a/pkg/controllers/dnspolicy/dnspolicy_healthchecks.go b/pkg/controllers/dnspolicy/dnspolicy_healthchecks.go index f22971fa6..cd3d1f91e 100644 --- a/pkg/controllers/dnspolicy/dnspolicy_healthchecks.go +++ b/pkg/controllers/dnspolicy/dnspolicy_healthchecks.go @@ -3,11 +3,10 @@ package dnspolicy import ( "context" "fmt" + "strings" "time" - "github.com/miekg/dns" - - "k8s.io/apimachinery/pkg/api/errors" + k8serror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,18 +19,13 @@ import ( "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" ) -var ( - defaultPort = 443 -) - func (r *DNSPolicyReconciler) reconcileHealthChecks(ctx context.Context, dnsPolicy *v1alpha1.DNSPolicy, gwDiffObj *reconcilers.GatewayDiff) error { log := crlog.FromContext(ctx) + log.V(3).Info("reconciling health checks") for _, gw := range append(gwDiffObj.GatewaysWithValidPolicyRef, gwDiffObj.GatewaysMissingPolicyRef...) { - expectedProbes, err := r.expectedProbesForGateway(ctx, gw, dnsPolicy) - if err != nil { - return fmt.Errorf("error generating probes for gateway %v: %w", gw.Gateway.Name, err) - } + log.V(3).Info("reconciling probes", "gateway", gw.Name) + expectedProbes := r.expectedProbesForGateway(ctx, gw, dnsPolicy) if err := r.createOrUpdateProbes(ctx, expectedProbes); err != nil { return fmt.Errorf("error creating and updating expected proves for gateway %v: %w", gw.Gateway.Name, err) } @@ -55,7 +49,7 @@ func (r *DNSPolicyReconciler) createOrUpdateProbes(ctx context.Context, expected //create or update all expected probes for _, hcProbe := range expectedProbes { p := &v1alpha1.DNSHealthCheckProbe{} - if err := r.Client().Get(ctx, client.ObjectKeyFromObject(hcProbe), p); errors.IsNotFound(err) { + if err := r.Client().Get(ctx, client.ObjectKeyFromObject(hcProbe), p); k8serror.IsNotFound(err) { if err := r.Client().Create(ctx, hcProbe); err != nil { return err } @@ -94,12 +88,12 @@ func (r *DNSPolicyReconciler) deleteUnexpectedGatewayProbes(ctx context.Context, return nil } -func (r *DNSPolicyReconciler) expectedProbesForGateway(ctx context.Context, gw common.GatewayWrapper, dnsPolicy *v1alpha1.DNSPolicy) ([]*v1alpha1.DNSHealthCheckProbe, error) { +func (r *DNSPolicyReconciler) expectedProbesForGateway(ctx context.Context, gw common.GatewayWrapper, dnsPolicy *v1alpha1.DNSPolicy) []*v1alpha1.DNSHealthCheckProbe { log := crlog.FromContext(ctx) var healthChecks []*v1alpha1.DNSHealthCheckProbe if dnsPolicy.Spec.HealthCheck == nil { log.V(3).Info("DNS Policy has no defined health check") - return nil, nil + return healthChecks } interval := metav1.Duration{Duration: 60 * time.Second} @@ -107,57 +101,54 @@ func (r *DNSPolicyReconciler) expectedProbesForGateway(ctx context.Context, gw c interval = *dnsPolicy.Spec.HealthCheck.Interval } - for _, listener := range gw.Spec.Listeners { + for _, address := range gw.Status.Addresses { + matches := strings.Split(address.Value, "/") + if len(matches) != 2 { + log.V(3).Info(fmt.Sprintf("%s cannot be processed. expected /", address.Value)) + return nil + } - log.V(3).Info("getting dnsrecord", "name", dnsRecordName(gw.Name, string(listener.Name))) - dnsRecord := &v1alpha1.DNSRecord{} - if err := r.Client().Get(ctx, client.ObjectKey{Name: dnsRecordName(gw.Name, string(listener.Name)), Namespace: dnsPolicy.Namespace}, dnsRecord); err != nil { - if errors.IsNotFound(err) { + for _, listener := range gw.Spec.Listeners { + if strings.Contains(string(*listener.Hostname), "*") { continue - } else { - return nil, err } - } - for _, endpoint := range dnsRecord.Spec.Endpoints { - log.V(1).Info("reconcileHealthChecks: found DNS Record endpoint", "endpoint", endpoint) - if endpoint.RecordType == string(v1alpha1.CNAMERecordType) { - // if the CNAME is a subdomain of this dnsrecord, skip it - if dns.IsSubDomain(dnsRecord.Name, endpoint.Targets[0]) { - continue - } - } else if endpoint.RecordType != string(v1alpha1.ARecordType) { - log.V(1).Info("reconcileHealthChecks: not an A record, skipping") - continue + port := dnsPolicy.Spec.HealthCheck.Port + if port == nil { + listenerPort := int(listener.Port) + port = &listenerPort } - for _, target := range endpoint.Targets { - port := dnsPolicy.Spec.HealthCheck.Port - if port == nil { - port = &defaultPort - } - log.V(1).Info("reconcileHealthChecks: adding health check for target", "target", target) - healthCheck := &v1alpha1.DNSHealthCheckProbe{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", target, dnsRecord.Name), - Namespace: gw.Namespace, - Labels: commonDNSRecordLabels(client.ObjectKeyFromObject(gw), client.ObjectKeyFromObject(dnsPolicy)), - }, - Spec: v1alpha1.DNSHealthCheckProbeSpec{ - Port: *port, - Host: dnsRecord.Name, - Address: target, - Path: dnsPolicy.Spec.HealthCheck.Endpoint, - Protocol: *dnsPolicy.Spec.HealthCheck.Protocol, - Interval: interval, - AdditionalHeadersRef: dnsPolicy.Spec.HealthCheck.AdditionalHeadersRef, - FailureThreshold: dnsPolicy.Spec.HealthCheck.FailureThreshold, - ExpectedResponses: dnsPolicy.Spec.HealthCheck.ExpectedResponses, - AllowInsecureCertificate: dnsPolicy.Spec.HealthCheck.AllowInsecureCertificates, - }, - } - healthChecks = append(healthChecks, withDNSRecord(dnsRecord, healthCheck)) + + // handle protocol being nil + var protocol string + if dnsPolicy.Spec.HealthCheck.Protocol == nil { + protocol = string(listener.Protocol) + } else { + protocol = string(*dnsPolicy.Spec.HealthCheck.Protocol) + } + log.V(1).Info("reconcileHealthChecks: adding health check for target", "target", address.Value) + healthCheck := &v1alpha1.DNSHealthCheckProbe{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s", matches[1], dnsPolicy.Name, listener.Name), + Namespace: gw.Namespace, + Labels: commonDNSRecordLabels(client.ObjectKeyFromObject(gw), client.ObjectKeyFromObject(dnsPolicy)), + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + Port: *port, + Host: string(*listener.Hostname), + Address: matches[1], + Path: dnsPolicy.Spec.HealthCheck.Endpoint, + Protocol: v1alpha1.HealthProtocol(protocol), + Interval: interval, + AdditionalHeadersRef: dnsPolicy.Spec.HealthCheck.AdditionalHeadersRef, + FailureThreshold: dnsPolicy.Spec.HealthCheck.FailureThreshold, + ExpectedResponses: dnsPolicy.Spec.HealthCheck.ExpectedResponses, + AllowInsecureCertificate: dnsPolicy.Spec.HealthCheck.AllowInsecureCertificates, + }, } + healthChecks = append(healthChecks, withGatewayListener(gw, listener, healthCheck)) } } - return healthChecks, nil + + return healthChecks } diff --git a/pkg/controllers/dnspolicy/dnspolicy_healthchecks_test.go b/pkg/controllers/dnspolicy/dnspolicy_healthchecks_test.go new file mode 100644 index 000000000..5574b014c --- /dev/null +++ b/pkg/controllers/dnspolicy/dnspolicy_healthchecks_test.go @@ -0,0 +1,353 @@ +package dnspolicy + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatewayapiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/kuadrant/kuadrant-operator/pkg/common" + "github.com/kuadrant/kuadrant-operator/pkg/reconcilers" + + "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" + testutil "github.com/Kuadrant/multicluster-gateway-controller/test/util" +) + +const ( + Domain = "thecat.com" + ValidTestHostname = "boop." + Domain + ValidTestWildcard = "*." + Domain +) + +func TestDNSPolicyReconciler_expectedProbesForGateway(t *testing.T) { + + type fields struct { + TargetRefReconciler reconcilers.TargetRefReconciler + DNSProvider dns.DNSProviderFactory + dnsHelper dnsHelper + Placer gateway.GatewayPlacer + } + type args struct { + ctx context.Context + gw common.GatewayWrapper + dnsPolicy *v1alpha1.DNSPolicy + } + tests := []struct { + name string + fields fields + args args + want []*v1alpha1.DNSHealthCheckProbe + }{ + { + name: "expected probes not nil when all values specified in the check", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "testgateway", + Namespace: "testnamespace", + }, + Spec: v1alpha2.GatewaySpec{ + Listeners: []v1alpha2.Listener{ + { + Name: "testlistener", + Hostname: (*gatewayapiv1beta1.Hostname)(testutil.Pointer(ValidTestHostname)), + }, + }, + }, + Status: v1alpha2.GatewayStatus{ + Addresses: []v1alpha2.GatewayAddress{ + { + Type: testutil.Pointer(gatewayapiv1beta1.IPAddressType), + Value: "clusterName/172.31.200.0", + }, + }, + }, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "testdnspolicy", + Namespace: "testnamespace", + }, + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{ + Endpoint: "/", + Port: testutil.Pointer(8443), + Protocol: testutil.Pointer(v1alpha1.HttpsProtocol), + AdditionalHeadersRef: &v1alpha1.AdditionalHeadersRef{ + Name: "probe-headers", + }, + FailureThreshold: testutil.Pointer(1), + ExpectedResponses: []int{ + 200, 201, + }, + AllowInsecureCertificates: true, + }, + }, + }, + }, + want: []*v1alpha1.DNSHealthCheckProbe{ + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "172.31.200.0-testdnspolicy-testlistener", + Namespace: "testnamespace", + Labels: map[string]string{ + DNSPolicyBackRefAnnotation: "testdnspolicy", + fmt.Sprintf("%s-namespace", DNSPolicyBackRefAnnotation): "testnamespace", + LabelGatewayNSRef: "testnamespace", + LabelGatewayReference: "testgateway", + }, + Annotations: map[string]string{ + "dnsrecord-name": "testgateway-testlistener", + "dnsrecord-namespace": "testnamespace", + }, + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + Port: 8443, + Host: ValidTestHostname, + Address: "172.31.200.0", + Path: "/", + Protocol: v1alpha1.HttpsProtocol, + Interval: metav1.Duration{Duration: 60 * time.Second}, + AdditionalHeadersRef: &v1alpha1.AdditionalHeadersRef{ + Name: "probe-headers", + }, + AllowInsecureCertificate: true, + ExpectedResponses: []int{200, 201}, + FailureThreshold: testutil.Pointer(1), + }, + }, + }, + }, + { + name: "expected probes not nil when some values not specified in the check", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "testgateway", + Namespace: "testnamespace", + }, + Spec: v1alpha2.GatewaySpec{ + Listeners: []v1alpha2.Listener{ + { + Name: "testlistener", + Hostname: (*gatewayapiv1beta1.Hostname)(testutil.Pointer(ValidTestHostname)), + Port: 443, + Protocol: gatewayapiv1beta1.ProtocolType(v1alpha1.HttpsProtocol), + }, + }, + }, + Status: v1alpha2.GatewayStatus{ + Addresses: []v1alpha2.GatewayAddress{ + { + Type: testutil.Pointer(gatewayapiv1beta1.IPAddressType), + Value: "clusterName/172.31.200.0", + }, + }, + }, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "testdnspolicy", + Namespace: "testnamespace", + }, + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{}, + }, + }, + }, + want: []*v1alpha1.DNSHealthCheckProbe{ + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "172.31.200.0-testdnspolicy-testlistener", + Namespace: "testnamespace", + Labels: map[string]string{ + DNSPolicyBackRefAnnotation: "testdnspolicy", + fmt.Sprintf("%s-namespace", DNSPolicyBackRefAnnotation): "testnamespace", + LabelGatewayNSRef: "testnamespace", + LabelGatewayReference: "testgateway", + }, + Annotations: map[string]string{ + "dnsrecord-name": "testgateway-testlistener", + "dnsrecord-namespace": "testnamespace", + }, + }, + Spec: v1alpha1.DNSHealthCheckProbeSpec{ + Port: 443, + Host: ValidTestHostname, + Address: "172.31.200.0", + Protocol: v1alpha1.HttpsProtocol, + Interval: metav1.Duration{Duration: 60 * time.Second}, + }, + }, + }, + }, + { + name: "no probes when listener has a wildcard domain", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + Spec: v1alpha2.GatewaySpec{ + Listeners: []v1alpha2.Listener{ + { + Hostname: (*gatewayapiv1beta1.Hostname)(testutil.Pointer(ValidTestWildcard)), + }, + }, + }, + Status: v1alpha2.GatewayStatus{ + Addresses: []v1alpha2.GatewayAddress{ + { + Type: testutil.Pointer(gatewayapiv1beta1.IPAddressType), + Value: "clusterName/172.31.200.0", + }, + }, + }, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{}, + }, + }, + }, + want: nil, + }, + { + name: "no probes when address.Value doesn't contain /", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + Status: v1alpha2.GatewayStatus{ + Addresses: []v1alpha2.GatewayAddress{ + { + Type: testutil.Pointer(gatewayapiv1beta1.IPAddressType), + Value: "clusterName:172.31.200.0", + }, + }, + }, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{}, + }, + }, + }, + want: nil, + }, + { + name: "no probes when no listeners defined", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + Status: v1alpha2.GatewayStatus{ + Addresses: []v1alpha2.GatewayAddress{ + { + Type: testutil.Pointer(gatewayapiv1beta1.IPAddressType), + Value: "clusterName/172.31.200.0", + }, + }, + }, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{}, + }, + }, + }, + want: nil, + }, + { + name: "no probes when no address defined", + fields: fields{ + TargetRefReconciler: reconcilers.TargetRefReconciler{}, + DNSProvider: nil, + dnsHelper: dnsHelper{}, + Placer: nil, + }, + args: args{ + ctx: nil, + gw: common.GatewayWrapper{ + Gateway: &gatewayapiv1beta1.Gateway{ + Status: v1alpha2.GatewayStatus{}, + }, + }, + dnsPolicy: &v1alpha1.DNSPolicy{ + Spec: v1alpha1.DNSPolicySpec{ + HealthCheck: &v1alpha1.HealthCheckSpec{}, + }, + }, + }, + want: nil, + }, + { + name: "no probes when no healthcheck spec defined", + fields: fields{}, + args: args{ + dnsPolicy: &v1alpha1.DNSPolicy{}, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &DNSPolicyReconciler{ + TargetRefReconciler: tt.fields.TargetRefReconciler, + DNSProvider: tt.fields.DNSProvider, + dnsHelper: tt.fields.dnsHelper, + Placer: tt.fields.Placer, + } + got := r.expectedProbesForGateway(tt.args.ctx, tt.args.gw, tt.args.dnsPolicy) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("expectedProbesForGateway() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controllers/events/probe_eventmapper.go b/pkg/controllers/events/probe_eventmapper.go new file mode 100644 index 000000000..942ab49b7 --- /dev/null +++ b/pkg/controllers/events/probe_eventmapper.go @@ -0,0 +1,61 @@ +package events + +import ( + "fmt" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/Kuadrant/multicluster-gateway-controller/pkg/_internal/metadata" + "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" +) + +// ProbeEventMapper is an EventHandler that maps DNSHealthCheckProbe object events to policy events. +type ProbeEventMapper struct { + Logger logr.Logger + PolicyKind string + PolicyRef string +} + +func (p *ProbeEventMapper) MapToPolicy(obj client.Object) []reconcile.Request { + return p.mapToPolicyRequest(obj, p.PolicyRef, p.PolicyKind) +} + +func NewProbeEventMapper(logger logr.Logger, policyRef, policyKind string) *ProbeEventMapper { + return &ProbeEventMapper{ + Logger: logger.WithName("GatewayEventMapper"), + PolicyKind: policyKind, + PolicyRef: policyRef, + } +} + +func (p *ProbeEventMapper) mapToPolicyRequest(obj client.Object, policyRef, policyKind string) []reconcile.Request { + logger := p.Logger.V(3).WithValues("object", client.ObjectKeyFromObject(obj)) + probe, ok := obj.(*v1alpha1.DNSHealthCheckProbe) + if !ok { + logger.Info("mapToPolicyRequest:", "error", fmt.Sprintf("%T is not a *v1alpha1.DNSHealthCheckProbe", obj)) + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, 0) + + policyName := metadata.GetLabel(probe, policyRef) + if policyName == "" { + return requests + } + policyNamespace := metadata.GetLabel(probe, fmt.Sprintf("%s-namespace", policyRef)) + if policyNamespace == "" { + return requests + } + logger.Info("mapToPolicyRequest", policyKind, policyName) + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: policyName, + Namespace: policyNamespace, + }}) + + return requests +} diff --git a/pkg/controllers/gateway/gateway_controller.go b/pkg/controllers/gateway/gateway_controller.go index 5ab868f2c..96132694d 100644 --- a/pkg/controllers/gateway/gateway_controller.go +++ b/pkg/controllers/gateway/gateway_controller.go @@ -209,7 +209,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } else if *address.Type == gatewayv1beta1.HostnameAddressType { addressType = MultiClusterHostnameAddressType } else { - break // ignore address type. Unsupported for multi cluster gateway + continue // ignore address type gatewayv1beta1.NamedAddressType. Unsupported for multi cluster gateway } allAddresses = append(allAddresses, gatewayv1beta1.GatewayAddress{ Type: &addressType, @@ -254,7 +254,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } if requeue { - log.V(3).Info("requeuing gateay in ", "namespace", upstreamGateway.Namespace, "with name", upstreamGateway.Name) + log.V(3).Info("requeuing gateway in ", "namespace", upstreamGateway.Namespace, "with name", upstreamGateway.Name) return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 10}, reconcileErr } return ctrl.Result{}, reconcileErr diff --git a/test/integration/dnsheathcheckprobe_controller_test.go b/test/integration/dnsheathcheckprobe_controller_test.go index 04e8022ea..0b4d6cd78 100644 --- a/test/integration/dnsheathcheckprobe_controller_test.go +++ b/test/integration/dnsheathcheckprobe_controller_test.go @@ -60,7 +60,7 @@ var _ = Describe("DNSHealthCheckProbe controller", func() { GinkgoWriter.Print(probeObj) - Expect(probeObj.Status.Healthy).Should(BeTrue()) + Expect(*probeObj.Status.Healthy).Should(BeTrue()) Expect(probeObj.Status.LastCheckedAt).Should(Not(BeZero())) }) It("Should update health status to unhealthy", func() { @@ -91,7 +91,7 @@ var _ = Describe("DNSHealthCheckProbe controller", func() { return nil }, timeout+(time.Second*20), interval).Should(BeNil()) - Expect(probeObj.Status.Healthy).Should(BeFalse()) + Expect(*probeObj.Status.Healthy).Should(BeFalse()) Expect(probeObj.Status.Reason).Should(Equal("Status code: 500")) }) }) diff --git a/test/integration/dnspolicy_controller_test.go b/test/integration/dnspolicy_controller_test.go index 281ee3bd0..2c1bb0e99 100644 --- a/test/integration/dnspolicy_controller_test.go +++ b/test/integration/dnspolicy_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/Kuadrant/multicluster-gateway-controller/pkg/_internal/metadata" "github.com/Kuadrant/multicluster-gateway-controller/pkg/apis/v1alpha1" . "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/dnspolicy" + mgcgateway "github.com/Kuadrant/multicluster-gateway-controller/pkg/controllers/gateway" "github.com/Kuadrant/multicluster-gateway-controller/pkg/dns" ) @@ -56,13 +57,19 @@ func testBuildGatewayClass(gwClassName, ns string) *gatewayv1beta1.GatewayClass } } -func testBuildGateway(gwName, gwClassName, hostname, ns string) *gatewayv1beta1.Gateway { +func testBuildGateway(gwName, gwClassName, hostname, ns, dnspolicy string) *gatewayv1beta1.Gateway { typedHostname := gatewayv1beta1.Hostname(hostname) wildcardHost := gatewayv1beta1.Hostname(TestWildCardListenerHost) return &gatewayv1beta1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: gwName, Namespace: ns, + Annotations: map[string]string{ + DNSPoliciesBackRefAnnotation: fmt.Sprintf("[{\"Namespace\":\"%s\",\"Name\":\"%s\"}]", ns, dnspolicy), + }, + Labels: map[string]string{ + "cluster.open-cluster-management.io/placement": "GatewayControllerTest", + }, }, Spec: gatewayv1beta1.GatewaySpec{ GatewayClassName: gatewayv1beta1.ObjectName(gwClassName), @@ -84,7 +91,7 @@ func testBuildGateway(gwName, gwClassName, hostname, ns string) *gatewayv1beta1. } } -func testBuildDNSPolicyWithHealthCheck(policyName, gwName, ns string) *v1alpha1.DNSPolicy { +func testBuildDNSPolicyWithHealthCheck(policyName, gwName, ns string, threshold *int) *v1alpha1.DNSPolicy { typedNamespace := gatewayv1beta1.Namespace(ns) protocol := v1alpha1.HttpProtocol return &v1alpha1.DNSPolicy{ @@ -100,8 +107,9 @@ func testBuildDNSPolicyWithHealthCheck(policyName, gwName, ns string) *v1alpha1. Namespace: &typedNamespace, }, HealthCheck: &v1alpha1.HealthCheckSpec{ - Endpoint: "/", - Protocol: &protocol, + Endpoint: "/", + Protocol: &protocol, + FailureThreshold: threshold, }, LoadBalancing: &v1alpha1.LoadBalancingSpec{ Weighted: &v1alpha1.LoadBalancingWeighted{ @@ -191,7 +199,7 @@ var _ = Describe("DNSPolicy", Ordered, func() { var lbHash, dnsRecordName, wildcardDNSRecordName string BeforeEach(func() { - gateway = testBuildGateway(TestPlacedGatewayName, gatewayClass.Name, TestAttachedRouteName, testNamespace) + gateway = testBuildGateway(TestPlacedGatewayName, gatewayClass.Name, TestAttachedRouteName, testNamespace, "test-dns-policy") lbHash = dns.ToBase36hash(fmt.Sprintf("%s-%s", gateway.Name, gateway.Namespace)) dnsRecordName = fmt.Sprintf("%s-%s", TestPlacedGatewayName, TestAttachedRouteName) wildcardDNSRecordName = fmt.Sprintf("%s-%s", TestPlacedGatewayName, TestWildCardListenerName) @@ -201,11 +209,16 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, TestTimeoutMedium, TestRetryIntervalMedium).ShouldNot(HaveOccurred()) }) + AfterEach(func() { + err := k8sClient.Delete(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + }) + Context("weighted dnspolicy", func() { var dnsPolicy *v1alpha1.DNSPolicy BeforeEach(func() { - dnsPolicy = testBuildDNSPolicyWithHealthCheck("test-dns-policy", TestPlacedGatewayName, testNamespace) + dnsPolicy = testBuildDNSPolicyWithHealthCheck("test-dns-policy", TestPlacedGatewayName, testNamespace, nil) Expect(k8sClient.Create(ctx, dnsPolicy)).To(BeNil()) Eventually(func() error { //dns policy exists return k8sClient.Get(ctx, client.ObjectKey{Name: dnsPolicy.Name, Namespace: dnsPolicy.Namespace}, dnsPolicy) @@ -240,9 +253,9 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, }, { - DNSName: "16z1l1.lb-" + lbHash + ".test.example.com", + DNSName: "s07c46.lb-" + lbHash + ".test.example.com", Targets: []string{ - "172.0.0.3", + TestAttachedRouteAddressOne, }, RecordType: "A", SetIdentifier: "", @@ -251,10 +264,10 @@ var _ = Describe("DNSPolicy", Ordered, func() { { DNSName: "default.lb-" + lbHash + ".test.example.com", Targets: []string{ - "16z1l1.lb-" + lbHash + ".test.example.com", + "s07c46.lb-" + lbHash + ".test.example.com", }, RecordType: "CNAME", - SetIdentifier: "16z1l1.lb-" + lbHash + ".test.example.com", + SetIdentifier: "s07c46.lb-" + lbHash + ".test.example.com", RecordTTL: 60, ProviderSpecific: v1alpha1.ProviderSpecific{ { @@ -263,6 +276,30 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, }, }, + { + DNSName: "default.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "2w705o.lb-" + lbHash + ".test.example.com", + Targets: []string{ + TestAttachedRouteAddressTwo, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, } Eventually(func() error { // DNS record exists if err := k8sClient.Get(ctx, client.ObjectKey{Name: dnsRecordName, Namespace: testNamespace}, createdDNSRecord); err != nil { @@ -274,6 +311,7 @@ var _ = Describe("DNSPolicy", Ordered, func() { return nil }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) Expect(createdDNSRecord.Spec.ManagedZoneRef.Name).To(Equal("example.com")) + Expect(createdDNSRecord.Spec.Endpoints).To(HaveLen(6)) Expect(createdDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) }) It("should create a wildcard dns record", func() { @@ -304,9 +342,9 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, }, { - DNSName: "16z1l1.lb-" + lbHash + ".example.com", + DNSName: "s07c46.lb-" + lbHash + ".example.com", Targets: []string{ - "172.0.0.3", + TestAttachedRouteAddressOne, }, RecordType: "A", SetIdentifier: "", @@ -315,10 +353,34 @@ var _ = Describe("DNSPolicy", Ordered, func() { { DNSName: "default.lb-" + lbHash + ".example.com", Targets: []string{ - "16z1l1.lb-" + lbHash + ".example.com", + "s07c46.lb-" + lbHash + ".example.com", }, RecordType: "CNAME", - SetIdentifier: "16z1l1.lb-" + lbHash + ".example.com", + SetIdentifier: "s07c46.lb-" + lbHash + ".example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "2w705o.lb-" + lbHash + ".example.com", + Targets: []string{ + TestAttachedRouteAddressTwo, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, + { + DNSName: "default.lb-" + lbHash + ".example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".example.com", RecordTTL: 60, ProviderSpecific: v1alpha1.ProviderSpecific{ { @@ -338,7 +400,9 @@ var _ = Describe("DNSPolicy", Ordered, func() { return nil }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) Expect(wildcardDNSRecord.Spec.ManagedZoneRef.Name).To(Equal("example.com")) + Expect(wildcardDNSRecord.Spec.Endpoints).To(HaveLen(6)) Expect(wildcardDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) + Expect(expectedEndpoints).Should(ContainElements(wildcardDNSRecord.Spec.Endpoints)) }) It("should have correct status", func() { @@ -494,17 +558,52 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) }) + AfterEach(func() { + err := k8sClient.Delete(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + }) + It("should create a dns record", func() { createdDNSRecord := &v1alpha1.DNSRecord{} expectedEndpoints := []*v1alpha1.Endpoint{ { - DNSName: "test.example.com", + DNSName: "2w705o.lb-" + lbHash + ".test.example.com", Targets: []string{ - "lb-" + lbHash + ".test.example.com", + TestAttachedRouteAddressTwo, }, - RecordType: "CNAME", + RecordType: "A", SetIdentifier: "", - RecordTTL: 300, + RecordTTL: 60, + }, + { + DNSName: "ie.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "ie.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "s07c46.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "s07c46.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, }, { DNSName: "lb-" + lbHash + ".test.example.com", @@ -537,28 +636,22 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, }, { - DNSName: "16z1l1.lb-" + lbHash + ".test.example.com", + DNSName: "s07c46.lb-" + lbHash + ".test.example.com", Targets: []string{ - "172.0.0.3", + TestAttachedRouteAddressOne, }, RecordType: "A", SetIdentifier: "", RecordTTL: 60, }, { - DNSName: "ie.lb-" + lbHash + ".test.example.com", + DNSName: "test.example.com", Targets: []string{ - "16z1l1.lb-" + lbHash + ".test.example.com", + "lb-" + lbHash + ".test.example.com", }, RecordType: "CNAME", - SetIdentifier: "16z1l1.lb-" + lbHash + ".test.example.com", - RecordTTL: 60, - ProviderSpecific: v1alpha1.ProviderSpecific{ - { - Name: "weight", - Value: "120", - }, - }, + SetIdentifier: "", + RecordTTL: 300, }, } Eventually(func() error { // DNS record exists @@ -571,14 +664,17 @@ var _ = Describe("DNSPolicy", Ordered, func() { return nil }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) Expect(createdDNSRecord.Spec.ManagedZoneRef.Name).To(Equal("example.com")) + Expect(createdDNSRecord.Spec.Endpoints).To(HaveLen(7)) Expect(createdDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) + Expect(expectedEndpoints).Should(ContainElements(createdDNSRecord.Spec.Endpoints)) + }) It("should create a wildcard dns record", func() { wildcardDNSRecord := &v1alpha1.DNSRecord{} expectedEndpoints := []*v1alpha1.Endpoint{ { - DNSName: TestWildCardListenerHost, + DNSName: "*.example.com", Targets: []string{ "lb-" + lbHash + ".example.com", }, @@ -586,6 +682,46 @@ var _ = Describe("DNSPolicy", Ordered, func() { SetIdentifier: "", RecordTTL: 300, }, + { + DNSName: "2w705o.lb-" + lbHash + ".example.com", + Targets: []string{ + TestAttachedRouteAddressTwo, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, + { + DNSName: "ie.lb-" + lbHash + ".example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".example.com", + RecordTTL: 60, + Labels: nil, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "ie.lb-" + lbHash + ".example.com", + Targets: []string{ + "s07c46.lb-" + lbHash + ".example.com", + }, + RecordType: "CNAME", + SetIdentifier: "s07c46.lb-" + lbHash + ".example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, { DNSName: "lb-" + lbHash + ".example.com", Targets: []string{ @@ -617,29 +753,14 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, }, { - DNSName: "16z1l1.lb-" + lbHash + ".example.com", + DNSName: "s07c46.lb-" + lbHash + ".example.com", Targets: []string{ - "172.0.0.3", + TestAttachedRouteAddressOne, }, RecordType: "A", SetIdentifier: "", RecordTTL: 60, }, - { - DNSName: "ie.lb-" + lbHash + ".example.com", - Targets: []string{ - "16z1l1.lb-" + lbHash + ".example.com", - }, - RecordType: "CNAME", - SetIdentifier: "16z1l1.lb-" + lbHash + ".example.com", - RecordTTL: 60, - ProviderSpecific: v1alpha1.ProviderSpecific{ - { - Name: "weight", - Value: "120", - }, - }, - }, } Eventually(func() error { // DNS record exists if err := k8sClient.Get(ctx, client.ObjectKey{Name: wildcardDNSRecordName, Namespace: dnsPolicy.Namespace}, wildcardDNSRecord); err != nil { @@ -651,7 +772,9 @@ var _ = Describe("DNSPolicy", Ordered, func() { return nil }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) Expect(wildcardDNSRecord.Spec.ManagedZoneRef.Name).To(Equal("example.com")) + Expect(wildcardDNSRecord.Spec.Endpoints).To(HaveLen(7)) Expect(wildcardDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) + Expect(expectedEndpoints).Should(ContainElements(wildcardDNSRecord.Spec.Endpoints)) }) }) }) @@ -662,8 +785,8 @@ var _ = Describe("DNSPolicy", Ordered, func() { testGatewayName := "test-not-placed-gateway" BeforeEach(func() { - gateway = testBuildGateway(testGatewayName, gatewayClass.Name, TestAttachedRouteName, testNamespace) - dnsPolicy = testBuildDNSPolicyWithHealthCheck("test-dns-policy", testGatewayName, testNamespace) + gateway = testBuildGateway(testGatewayName, gatewayClass.Name, TestAttachedRouteName, testNamespace, "test-dns-policy") + dnsPolicy = testBuildDNSPolicyWithHealthCheck("test-dns-policy", testGatewayName, testNamespace, nil) Expect(k8sClient.Create(ctx, gateway)).To(BeNil()) Expect(k8sClient.Create(ctx, dnsPolicy)).To(BeNil()) @@ -690,7 +813,7 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) }) - AfterAll(func() { + AfterEach(func() { err := k8sClient.Delete(ctx, gateway) Expect(err).ToNot(HaveOccurred()) }) @@ -759,4 +882,343 @@ var _ = Describe("DNSPolicy", Ordered, func() { }, time.Second*5, time.Second).Should(BeNil()) }) }) + + Context("probes status impact DNS records", func() { + var gateway *gatewayv1beta1.Gateway + var dnsRecordName, lbHash string + var dnsPolicy *v1alpha1.DNSPolicy + var unhealthy bool + + BeforeEach(func() { + gateway = testBuildGateway(TestPlacedGatewayName, gatewayClass.Name, TestAttachedRouteName, testNamespace, "test-dns-policy") + dnsRecordName = fmt.Sprintf("%s-%s", TestPlacedGatewayName, TestAttachedRouteName) + Expect(k8sClient.Create(ctx, gateway)).To(BeNil()) + Eventually(func() error { //gateway exists + if err := k8sClient.Get(ctx, client.ObjectKey{Name: gateway.Name, Namespace: gateway.Namespace}, gateway); err != nil { + return err + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + + threshold := 4 + dnsPolicy = testBuildDNSPolicyWithHealthCheck("test-dns-policy", TestPlacedGatewayName, testNamespace, &threshold) + Expect(k8sClient.Create(ctx, dnsPolicy)).To(BeNil()) + Eventually(func() error { //dns policy exists + if err := k8sClient.Get(ctx, client.ObjectKey{Name: dnsPolicy.Name, Namespace: dnsPolicy.Namespace}, dnsPolicy); err != nil { + return err + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + }) + + AfterEach(func() { + //clean up gateway + gatewayList := &gatewayv1beta1.GatewayList{} + Expect(k8sClient.List(ctx, gatewayList)).To(BeNil()) + for _, gw := range gatewayList.Items { + k8sClient.Delete(ctx, &gw) + } + }) + + It("should create a dns record", func() { + createdDNSRecord := &v1alpha1.DNSRecord{} + Eventually(func() error { // DNS record exists + if err := k8sClient.Get(ctx, client.ObjectKey{Name: dnsRecordName, Namespace: testNamespace}, createdDNSRecord); err != nil { + return err + } + if len(createdDNSRecord.Spec.Endpoints) != 6 { + return fmt.Errorf("expected %v endpoints in DNSRecord, got %v", 6, len(createdDNSRecord.Spec.Endpoints)) + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + }) + It("should have probes that are healthy", func() { + err := k8sClient.Get(ctx, client.ObjectKey{Name: gateway.Name, Namespace: gateway.Namespace}, gateway) + Expect(err).NotTo(HaveOccurred()) + patch := client.MergeFrom(gateway.DeepCopy()) + addressType := mgcgateway.MultiClusterIPAddressType + gateway.Status.Addresses = []gatewayv1beta1.GatewayAddress{ + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressOne), + }, + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressTwo), + }, + } + Expect(k8sClient.Status().Patch(ctx, gateway, patch)).To(BeNil()) + + probeList := &v1alpha1.DNSHealthCheckProbeList{} + Eventually(func() error { + Expect(k8sClient.List(ctx, probeList, &client.ListOptions{Namespace: testNamespace})).To(BeNil()) + if len(probeList.Items) != 2 { + return fmt.Errorf("expected %v probes, got %v", 2, len(probeList.Items)) + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + Expect(len(probeList.Items)).To(Equal(2)) + }) + + Context("all unhealthy probes", func() { + It("should publish all dns records endpoints", func() { + lbHash = dns.ToBase36hash(fmt.Sprintf("%s-%s", gateway.Name, gateway.Namespace)) + + expectedEndpoints := []*v1alpha1.Endpoint{ + { + DNSName: "2w705o.lb-" + lbHash + ".test.example.com", + Targets: []string{ + TestAttachedRouteAddressTwo, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, + { + DNSName: "s07c46.lb-" + lbHash + ".test.example.com", + Targets: []string{ + TestAttachedRouteAddressOne, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, + { + DNSName: "default.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "default.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "s07c46.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "s07c46.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + Labels: nil, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-" + lbHash + ".test.example.com", + Targets: []string{ + "default.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: 300, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "test.example.com", + Targets: []string{ + "lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "", + RecordTTL: 300, + }, + } + + err := k8sClient.Get(ctx, client.ObjectKey{Name: gateway.Name, Namespace: gateway.Namespace}, gateway) + Expect(err).NotTo(HaveOccurred()) + patch := client.MergeFrom(gateway.DeepCopy()) + addressType := mgcgateway.MultiClusterIPAddressType + gateway.Status.Addresses = []gatewayv1beta1.GatewayAddress{ + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressOne), + }, + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressTwo), + }, + } + Expect(k8sClient.Status().Patch(ctx, gateway, patch)).To(BeNil()) + + probeList := &v1alpha1.DNSHealthCheckProbeList{} + Eventually(func() error { + Expect(k8sClient.List(ctx, probeList, &client.ListOptions{Namespace: testNamespace})).To(BeNil()) + if len(probeList.Items) != 2 { + return fmt.Errorf("expected %v probes, got %v", 2, len(probeList.Items)) + } + return nil + }, TestTimeoutLong, TestRetryIntervalMedium).Should(BeNil()) + + for _, probe := range probeList.Items { + Eventually(func() error { + if probe.Name == fmt.Sprintf("%s-test-dns-policy-%s", TestAttachedRouteAddressTwo, TestAttachedRouteName) || + probe.Name == fmt.Sprintf("%s-test-dns-policy-%s", TestAttachedRouteAddressOne, TestAttachedRouteName) { + getProbe := &v1alpha1.DNSHealthCheckProbe{} + if err = k8sClient.Get(ctx, client.ObjectKey{Name: probe.Name, Namespace: probe.Namespace}, getProbe); err != nil { + return err + } + patch := client.MergeFrom(getProbe.DeepCopy()) + unhealthy = false + getProbe.Status = v1alpha1.DNSHealthCheckProbeStatus{ + LastCheckedAt: metav1.NewTime(time.Now()), + ConsecutiveFailures: *getProbe.Spec.FailureThreshold + 1, + Healthy: &unhealthy, + } + if err = k8sClient.Status().Patch(ctx, getProbe, patch); err != nil { + return err + } + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + } + createdDNSRecord := &v1alpha1.DNSRecord{} + Eventually(func() error { + err := k8sClient.Get(ctx, client.ObjectKey{Name: dnsRecordName, Namespace: testNamespace}, createdDNSRecord) + if err != nil && k8serrors.IsNotFound(err) { + return err + } + if len(createdDNSRecord.Spec.Endpoints) != len(expectedEndpoints) { + return fmt.Errorf("expected %v endpoints in DNSRecord, got %v", len(expectedEndpoints), len(createdDNSRecord.Spec.Endpoints)) + } + return nil + }, TestTimeoutLong, TestRetryIntervalMedium).Should(BeNil()) + Expect(createdDNSRecord.Spec.Endpoints).To(HaveLen(6)) + Expect(createdDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) + Expect(expectedEndpoints).Should(ContainElements(createdDNSRecord.Spec.Endpoints)) + + }) + }) + Context("some unhealthy endpoints", func() { + It("should publish expected endpoints", func() { + lbHash = dns.ToBase36hash(fmt.Sprintf("%s-%s", gateway.Name, gateway.Namespace)) + + expectedEndpoints := []*v1alpha1.Endpoint{ + { + DNSName: "2w705o.lb-" + lbHash + ".test.example.com", + Targets: []string{ + TestAttachedRouteAddressTwo, + }, + RecordType: "A", + SetIdentifier: "", + RecordTTL: 60, + }, + { + DNSName: "default.lb-" + lbHash + ".test.example.com", + Targets: []string{ + "2w705o.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "2w705o.lb-" + lbHash + ".test.example.com", + RecordTTL: 60, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "weight", + Value: "120", + }, + }, + }, + { + DNSName: "lb-" + lbHash + ".test.example.com", + Targets: []string{ + "default.lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "default", + RecordTTL: 300, + ProviderSpecific: v1alpha1.ProviderSpecific{ + { + Name: "geo-code", + Value: "*", + }, + }, + }, + { + DNSName: "test.example.com", + Targets: []string{ + "lb-" + lbHash + ".test.example.com", + }, + RecordType: "CNAME", + SetIdentifier: "", + RecordTTL: 300, + }, + } + + err := k8sClient.Get(ctx, client.ObjectKey{Name: gateway.Name, Namespace: gateway.Namespace}, gateway) + Expect(err).NotTo(HaveOccurred()) + patch := client.MergeFrom(gateway.DeepCopy()) + addressType := mgcgateway.MultiClusterIPAddressType + gateway.Status.Addresses = []gatewayv1beta1.GatewayAddress{ + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressOne), + }, + { + Type: &addressType, + Value: fmt.Sprintf("%s/%s", "kind-mgc-control-plane", TestAttachedRouteAddressTwo), + }, + } + Expect(k8sClient.Status().Patch(ctx, gateway, patch)).To(BeNil()) + + probeList := &v1alpha1.DNSHealthCheckProbeList{} + Eventually(func() error { + Expect(k8sClient.List(ctx, probeList, &client.ListOptions{Namespace: testNamespace})).To(BeNil()) + if len(probeList.Items) != 2 { + return fmt.Errorf("expected %v probes, got %v", 2, len(probeList.Items)) + } + return nil + }, TestTimeoutLong, TestRetryIntervalMedium).Should(BeNil()) + Expect(len(probeList.Items)).To(Equal(2)) + + Eventually(func() error { + getProbe := &v1alpha1.DNSHealthCheckProbe{} + if err = k8sClient.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("%s-test-dns-policy-%s", TestAttachedRouteAddressOne, TestAttachedRouteName), Namespace: testNamespace}, getProbe); err != nil { + return err + } + patch := client.MergeFrom(getProbe.DeepCopy()) + unhealthy = false + getProbe.Status = v1alpha1.DNSHealthCheckProbeStatus{ + LastCheckedAt: metav1.NewTime(time.Now()), + ConsecutiveFailures: *getProbe.Spec.FailureThreshold + 1, + Healthy: &unhealthy, + } + if err = k8sClient.Status().Patch(ctx, getProbe, patch); err != nil { + return err + } + return nil + }, TestTimeoutLong, TestRetryIntervalMedium).Should(BeNil()) + + // after that verify that in time the endpoints are 5 in the dnsrecord + createdDNSRecord := &v1alpha1.DNSRecord{} + Eventually(func() error { + err := k8sClient.Get(ctx, client.ObjectKey{Name: dnsRecordName, Namespace: testNamespace}, createdDNSRecord) + if err != nil && k8serrors.IsNotFound(err) { + return err + } + if len(createdDNSRecord.Spec.Endpoints) != len(expectedEndpoints) { + return fmt.Errorf("expected %v endpoints in DNSRecord, got %v", len(expectedEndpoints), len(createdDNSRecord.Spec.Endpoints)) + } + return nil + }, TestTimeoutMedium, TestRetryIntervalMedium).Should(BeNil()) + Expect(createdDNSRecord.Spec.Endpoints).To(HaveLen(4)) + Expect(createdDNSRecord.Spec.Endpoints).Should(ContainElements(expectedEndpoints)) + Expect(expectedEndpoints).Should(ContainElements(createdDNSRecord.Spec.Endpoints)) + }) + }) + }) }) diff --git a/test/integration/helper_test.go b/test/integration/helper_test.go index 3bdb91ecf..c2878ad34 100644 --- a/test/integration/helper_test.go +++ b/test/integration/helper_test.go @@ -24,43 +24,61 @@ import ( ) const ( - TestTimeoutMedium = time.Second * 10 - TestTimeoutLong = time.Second * 30 - ConsistentlyTimeoutMedium = time.Second * 60 - TestRetryIntervalMedium = time.Millisecond * 250 - TestPlacedGatewayName = "test-placed-gateway" - TestPlacedClusterName = "test-placed-cluster" - TestAttachedRouteName = "test.example.com" - TestWildCardListenerName = "wildcard" - TestWildCardListenerHost = "*.example.com" - TestAttachedRouteAddress = "172.0.0.3" - nsSpoke1Name = "test-spoke-cluster-1" - nsSpoke2Name = "test-spoke-cluster-2" - defaultNS = "default" - gatewayFinalizer = "kuadrant.io/gateway" - providerCredential = "secretname" + TestTimeoutMedium = time.Second * 10 + TestTimeoutLong = time.Second * 30 + ConsistentlyTimeoutMedium = time.Second * 60 + TestRetryIntervalMedium = time.Millisecond * 250 + TestPlacedGatewayName = "test-placed-gateway" + TestPlacedClusterControlName = "test-placed-control" + TestPlaceClusterWorkloadName = "test-placed-workload-1" + TestAttachedRouteName = "test.example.com" + TestWildCardListenerName = "wildcard" + TestWildCardListenerHost = "*.example.com" + TestAttachedRouteAddressOne = "172.0.0.1" + TestAttachedRouteAddressTwo = "172.0.0.2" + nsSpoke1Name = "test-spoke-cluster-1" + nsSpoke2Name = "test-spoke-cluster-2" + defaultNS = "default" + gatewayFinalizer = "kuadrant.io/gateway" + providerCredential = "secretname" ) -// FakeOCMPlacer has one gateway called `placedGatewayName` placed on one cluster called `placedClusterName` with one -// attached route called `attachedRouteName` with an address value of `attachedRouteAddress` -type FakeOCMPlacer struct { - placedGatewayName string - placedClusterName string - attachedRouteName string +// FakeOCMPlacer has one gateway called "test-placed-gateway" +// placed on two clusters called +// "test-placed-control" with address value of "172.0.0.3" and +// "test-placed-workload-1" with address value of "172.0.0.4" with one +// attached route "test.example.com" + +type placedClusters struct { + name string attachedRouteAddress string } -func NewFakeOCMPlacer(placedGatewayName, placedClusterName, attachedRouteName, attachedRouteAddress string) *FakeOCMPlacer { +type FakeOCMPlacer struct { + placedGatewayName string + placedClusters []placedClusters + attachedRouteName string +} + +func NewFakeOCMPlacer(placedGatewayName, attachedRouteName string) *FakeOCMPlacer { return &FakeOCMPlacer{ - placedGatewayName: placedGatewayName, - placedClusterName: placedClusterName, - attachedRouteName: attachedRouteName, - attachedRouteAddress: attachedRouteAddress, + placedGatewayName: placedGatewayName, + placedClusters: []placedClusters{ + { + name: TestPlacedClusterControlName, + attachedRouteAddress: TestAttachedRouteAddressOne, + }, + { + name: TestPlaceClusterWorkloadName, + attachedRouteAddress: TestAttachedRouteAddressTwo, + }, + }, + attachedRouteName: attachedRouteName, } } func NewTestOCMPlacer() *FakeOCMPlacer { - return NewFakeOCMPlacer(TestPlacedGatewayName, TestPlacedClusterName, TestAttachedRouteName, TestAttachedRouteAddress) + return NewFakeOCMPlacer(TestPlacedGatewayName, TestAttachedRouteName) } func (f FakeOCMPlacer) Place(ctx context.Context, upstream *gatewayv1beta1.Gateway, downstream *gatewayv1beta1.Gateway, children ...metav1.Object) (sets.Set[string], error) { @@ -69,8 +87,10 @@ func (f FakeOCMPlacer) Place(ctx context.Context, upstream *gatewayv1beta1.Gatew func (f FakeOCMPlacer) GetPlacedClusters(ctx context.Context, gateway *gatewayv1beta1.Gateway) (sets.Set[string], error) { clusters := sets.Set[string](sets.NewString()) - if gateway.Name == f.placedGatewayName { - clusters.Insert(f.placedClusterName) + for _, cluster := range f.placedClusters { + if gateway.Name == f.placedGatewayName { + clusters.Insert(cluster.name) + } } return clusters, nil } @@ -81,20 +101,24 @@ func (f FakeOCMPlacer) GetClusters(ctx context.Context, gateway *gatewayv1beta1. func (f FakeOCMPlacer) ListenerTotalAttachedRoutes(ctx context.Context, gateway *gatewayv1beta1.Gateway, listenerName string, downstream string) (int, error) { count := 0 - if gateway.Name == f.placedGatewayName && (listenerName == f.attachedRouteName || listenerName == TestWildCardListenerName) && downstream == f.placedClusterName { - count = 1 + for _, placedCluster := range f.placedClusters { + if gateway.Name == f.placedGatewayName && (listenerName == f.attachedRouteName || listenerName == TestWildCardListenerName) && downstream == placedCluster.name { + count = 1 + } } return count, nil } func (f FakeOCMPlacer) GetAddresses(ctx context.Context, gateway *gatewayv1beta1.Gateway, downstream string) ([]gatewayv1beta1.GatewayAddress, error) { gwAddresses := []gatewayv1beta1.GatewayAddress{} - if gateway.Name == f.placedGatewayName && downstream == f.placedClusterName { - t := gatewayv1beta1.IPAddressType - gwAddresses = append(gwAddresses, gatewayv1beta1.GatewayAddress{ - Type: &t, - Value: f.attachedRouteAddress, - }) + t := gatewayv1beta1.IPAddressType + for _, cluster := range f.placedClusters { + if gateway.Name == f.placedGatewayName && downstream == cluster.name { + gwAddresses = append(gwAddresses, gatewayv1beta1.GatewayAddress{ + Type: &t, + Value: cluster.attachedRouteAddress, + }) + } } return gwAddresses, nil } diff --git a/test/integration/suite_test.go b/test/integration/suite_test.go index a91ec2014..cd010ec48 100644 --- a/test/integration/suite_test.go +++ b/test/integration/suite_test.go @@ -155,7 +155,7 @@ var _ = BeforeSuite(func() { BaseReconciler: dnsPolicyBaseReconciler, }, DNSProvider: providerFactory, - Placement: testPlc, + Placer: testPlc, }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred())