From 25ab53f8c6a2ee5786b2a2f67b646475320b5d36 Mon Sep 17 00:00:00 2001 From: AmaliMatharaarachchi Date: Thu, 30 May 2024 00:04:29 +0530 Subject: [PATCH] add status update for httproute gateway --- .../internal/discovery/xds/common/utils.go | 51 +- .../discovery/xds/common/utils_test.go | 22 + adapter/internal/discovery/xds/server.go | 2 +- .../internal/oasparser/envoyconf/listener.go | 10 +- .../internal/operator/constants/constants.go | 3 + .../controllers/dp/gateway_controller.go | 269 ++++++++- .../controllers/dp/httproute_controller.go | 530 ++++++++++++++++++ adapter/internal/operator/operator.go | 3 + .../operator/synchronizer/gateway_state.go | 1 + .../gateway-components/log-conf.yaml | 2 +- .../ballerina/modules/model/HttpRoute.bal | 1 + .../wso2/apk/integration/api/BaseSteps.java | 7 +- .../tests/api/APIDefinitionEndpoint.feature | 4 +- .../tests/api/GlobalInterceptor.feature | 2 +- .../resources/tests/api/Interceptor.feature | 18 +- 15 files changed, 876 insertions(+), 49 deletions(-) create mode 100644 adapter/internal/operator/controllers/dp/httproute_controller.go diff --git a/adapter/internal/discovery/xds/common/utils.go b/adapter/internal/discovery/xds/common/utils.go index b09cdb4113..63574eef30 100644 --- a/adapter/internal/discovery/xds/common/utils.go +++ b/adapter/internal/discovery/xds/common/utils.go @@ -19,11 +19,13 @@ package common import ( - "sync" "fmt" "regexp" "strings" + "sync" + discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const nodeIDArrayMaxLength int = 20 @@ -38,12 +40,13 @@ type NodeQueue struct { // CheckEntryAndSwapToEnd function does the following. Recently accessed entry is removed last. // Array should have a maximum length. If the the provided nodeId may or may not be within the array. // -// 1. If the array's maximum length is not reached after adding the new element and the element is not inside the array, -// append the element to the end. -// 2. If the array is at maximum length and element is not within the array, the new entry should be appended to the end -// and the 0th element should be removed. -// 3. If the array is at the maximum length and element is inside the array, the new element should be appended and the already -// existing entry should be removed from the position. +// 1. If the array's maximum length is not reached after adding the new element and the element is not inside the array, +// append the element to the end. +// 2. If the array is at maximum length and element is not within the array, the new entry should be appended to the end +// and the 0th element should be removed. +// 3. If the array is at the maximum length and element is inside the array, the new element should be appended and the already +// existing entry should be removed from the position. +// // Returns the modified array and true if the entry is a new addition. func (nodeQueue *NodeQueue) checkEntryAndMoveToEnd(nodeID string) (isNewAddition bool) { matchedIndex := -1 @@ -131,3 +134,37 @@ func MatchesHostname(domain, pattern string) bool { return matched } + +// PointerCopy returns a pointer to a new memory location containing a copy of the input value. +func PointerCopy[T any](x T) *T { + return &x +} + +// AreConditionsSame checks if two metav1.Condition objects have the same attributes. +// It returns true if all the attributes (Type, Status, Reason, Message, ObservedGeneration) +// are equal, otherwise it returns false. +func AreConditionsSame(condition1 metav1.Condition, condition2 metav1.Condition) bool { + return condition1.Type == condition2.Type && + condition1.Status == condition2.Status && + condition1.Reason == condition2.Reason && + condition1.Message == condition2.Message && + condition1.ObservedGeneration == condition2.ObservedGeneration +} + +// BothListContainsSameConditions checks if two lists of metav1.Conditions contain the same conditions. +// It returns true if all conditions in conditionList1 are found in conditionList2. +func BothListContainsSameConditions(conditionList1 []metav1.Condition, conditionList2 []metav1.Condition) bool { + for _, condition1 := range conditionList1 { + flag := false + for _, condition2 := range conditionList2 { + flag = AreConditionsSame(condition1, condition2) + if flag { + continue + } + } + if !flag { + return false + } + } + return true +} diff --git a/adapter/internal/discovery/xds/common/utils_test.go b/adapter/internal/discovery/xds/common/utils_test.go index 42bb0e3536..07bf52aee1 100644 --- a/adapter/internal/discovery/xds/common/utils_test.go +++ b/adapter/internal/discovery/xds/common/utils_test.go @@ -73,3 +73,25 @@ func generateNodeArray(length int) []string { } return array } + +func TestMatchDomain(t *testing.T) { + tests := []struct { + domain string + pattern string + expected bool + }{ + {"test.google.com", "*.google.com", true}, + {"test.google2.com", "*.google.com", false}, + {"sub.test.google.com", "*.google.com", true}, + {"sub.test.google2.com", "*.google.com", false}, + } + + for _, tt := range tests { + t.Run(tt.domain, func(t *testing.T) { + actual := MatchesHostname(tt.domain, tt.pattern) + if actual != tt.expected { + t.Errorf("Expected %v, but got %v", tt.expected, actual) + } + }) + } +} diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index 1f1cd89d26..ac54994366 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -349,7 +349,7 @@ func GenerateEnvoyResoucesForGateway(gatewayName string) ([]types.Resource, // Find gateway listeners that has $systemHost as its hostname and add the system routeConfig referencing those listeners gatewayListeners := dataholder.GetAllGatewayListenerSections() for _, listener := range gatewayListeners { - if systemHost == string(*listener.Hostname) { + if listener.Hostname != nil && systemHost == string(*listener.Hostname) { var vhostToRouteArrayFilteredMapForSystemEndpoints = make(map[string][]*routev3.Route) vhostToRouteArrayFilteredMapForSystemEndpoints[systemHost] = vhostToRouteArrayMap[systemHost] routeConfigName := common.GetEnvoyRouteConfigName(common.GetEnvoyListenerName(string(listener.Protocol), uint32(listener.Port)), string(listener.Name)) diff --git a/adapter/internal/oasparser/envoyconf/listener.go b/adapter/internal/oasparser/envoyconf/listener.go index 7237a13d24..46e933e551 100644 --- a/adapter/internal/oasparser/envoyconf/listener.go +++ b/adapter/internal/oasparser/envoyconf/listener.go @@ -87,7 +87,7 @@ func CreateListenerByGateway(gateway *gwapiv1.Gateway, resolvedListenerCerts map } protocolListenerMap[listener.Protocol][port] = append(protocolListenerMap[listener.Protocol][port], listener) } - loggers.LoggerAPKOperator.Infof("CreateListenerByGateway is called. ProtocolListenerMap: %+v", protocolListenerMap) + loggers.LoggerAPKOperator.Debugf("CreateListenerByGateway is called. ProtocolListenerMap: %+v, gateway %+v", protocolListenerMap, gateway.Spec.Listeners) listenerList := make([]*listenerv3.Listener, 0) for protocol, protocolPort := range protocolListenerMap { for port, listeners := range protocolPort { @@ -109,9 +109,13 @@ func CreateListenerByGateway(gateway *gwapiv1.Gateway, resolvedListenerCerts map end`) } listenerName = common.GetEnvoyListenerName(string(protocol), port) - filterChainMatch := &listenerv3.FilterChainMatch{ - ServerNames: []string{string(*listenerObj.Hostname)}, + var filterChainMatch *listenerv3.FilterChainMatch + if listenerObj.Hostname != nil { + filterChainMatch = &listenerv3.FilterChainMatch{ + ServerNames: []string{string(*listenerObj.Hostname)}, + } } + var transportSocket *corev3.TransportSocket if protocol == gwapiv1.HTTPSProtocolType { publicCertData := resolvedListenerCerts[string(listenerObj.Name)]["tls.crt"] diff --git a/adapter/internal/operator/constants/constants.go b/adapter/internal/operator/constants/constants.go index 872c3491e3..10f69ebfc5 100644 --- a/adapter/internal/operator/constants/constants.go +++ b/adapter/internal/operator/constants/constants.go @@ -24,6 +24,8 @@ const ( ApplicationController string = "ApplicationController" SubscriptionController string = "SubscriptionController" TokenIssuerController string = "TokenIssuerController" + HTTPRouteController string = "HttpRouteController" + GatewayClassController string = "GatewayClassController" ) // API events related constants @@ -31,6 +33,7 @@ const ( Create string = "CREATED" Update string = "UPDATED" Delete string = "DELETED" + Accept string = "Accepted" ) // Environment variable names and default values diff --git a/adapter/internal/operator/controllers/dp/gateway_controller.go b/adapter/internal/operator/controllers/dp/gateway_controller.go index 476cbfdff3..4ee2356197 100644 --- a/adapter/internal/operator/controllers/dp/gateway_controller.go +++ b/adapter/internal/operator/controllers/dp/gateway_controller.go @@ -24,6 +24,7 @@ import ( "github.com/wso2/apk/adapter/config" "github.com/wso2/apk/adapter/internal/discovery/xds" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" "github.com/wso2/apk/adapter/internal/loggers" "github.com/wso2/apk/adapter/pkg/logging" "golang.org/x/exp/maps" @@ -56,9 +57,16 @@ const ( ) var ( - setReadiness sync.Once + setReadiness sync.Once + supportedKinds = []gwapiv1.Kind{gwapiv1.Kind("HTTPRoute")} + controllerName = "wso2.com/apk-envoy" ) +// GetControllerName returns the controller name that supported by APK +func GetControllerName() string { + return controllerName +} + // GatewayReconciler reconciles a Gateway object type GatewayReconciler struct { client k8client.Client @@ -141,6 +149,12 @@ func NewGatewayController(mgr manager.Manager, operatorDataStore *synchronizer.O return err } + if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.HTTPRoute{}), + handler.EnqueueRequestsFromMapFunc(r.getHTTPRoutes), predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3121, logging.BLOCKER, "Error watching HttpRoutes resources: %v", err)) + return err + } + loggers.LoggerAPKOperator.Info("Gateway Controller successfully started. Watching Gateway Objects....") return nil } @@ -160,7 +174,7 @@ func NewGatewayController(mgr manager.Manager, operatorDataStore *synchronizer.O // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (gatewayReconciler *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Check whether the Gateway CR exist, if not consider as a DELETE event. - loggers.LoggerAPKOperator.Infof("Reconciling gateway...") + loggers.LoggerAPKOperator.Infof("Reconciling gateway... %s", req.NamespacedName.String()) var gatewayDef gwapiv1.Gateway if err := gatewayReconciler.client.Get(ctx, req.NamespacedName, &gatewayDef); err != nil { gatewayState, found := gatewayReconciler.ods.GetCachedGateway(req.NamespacedName) @@ -176,20 +190,34 @@ func (gatewayReconciler *GatewayReconciler) Reconcile(ctx context.Context, req c } var gwCondition []metav1.Condition = gatewayDef.Status.Conditions - gatewayStateData, err := gatewayReconciler.resolveGatewayState(ctx, gatewayDef) + gatewayStateData, listenerStatuses, err := gatewayReconciler.resolveGatewayState(ctx, gatewayDef) + // Check whether the status change is needed for gateway + statusChanged := isStatusChanged(gatewayDef, listenerStatuses) + loggers.LoggerAPKOperator.Infof("Status changed ? %+v", statusChanged) + if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3122, logging.BLOCKER, "Error resolving Gateway State %s: %v", req.NamespacedName.String(), err)) return ctrl.Result{}, err } + state := constants.Update + var ( + events = make([]string, 0) + updated = false + cachedGateway synchronizer.GatewayState + ) - if gwCondition[0].Type != "Accepted" { + if gwCondition[0].Status != metav1.ConditionTrue { gatewayState := gatewayReconciler.ods.AddGatewayState(gatewayDef, gatewayStateData) *gatewayReconciler.ch <- synchronizer.GatewayEvent{EventType: constants.Create, Event: gatewayState} - gatewayReconciler.handleGatewayStatus(req.NamespacedName, constants.Create, []string{}) - } else if cachedGateway, events, updated := + state = constants.Create + } else if cachedGateway, events, updated = gatewayReconciler.ods.UpdateGatewayState(&gatewayDef, gatewayStateData); updated { *gatewayReconciler.ch <- synchronizer.GatewayEvent{EventType: constants.Update, Event: cachedGateway} - gatewayReconciler.handleGatewayStatus(req.NamespacedName, constants.Update, events) + state = constants.Update + } + if statusChanged || updated { + loggers.LoggerAPKOperator.Infof("Updating gateway status. Gateway: %s", utils.NamespacedName(&gatewayDef)) + gatewayReconciler.handleGatewayStatus(req.NamespacedName, state, events, listenerStatuses) } setReadiness.Do(gatewayReconciler.setGatewayReadiness) return ctrl.Result{}, nil @@ -217,29 +245,100 @@ func (gatewayReconciler *GatewayReconciler) resolveListenerSecretRefs(ctx contex // resolveGatewayState resolves the GatewayState struct using gwapiv1.Gateway and resource indexes func (gatewayReconciler *GatewayReconciler) resolveGatewayState(ctx context.Context, - gateway gwapiv1.Gateway) (*synchronizer.GatewayStateData, error) { + gateway gwapiv1.Gateway) (*synchronizer.GatewayStateData, []gwapiv1.ListenerStatus, error) { gatewayState := &synchronizer.GatewayStateData{} var err error resolvedListenerCerts := make(map[string]map[string][]byte) namespace := gwapiv1.Namespace(gateway.Namespace) + listenerstatuses := make([]gwapiv1.ListenerStatus, 0) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3124, logging.MAJOR, "Error while getting http routes: %s", err)) + } // Retireve listener Certificates for _, listener := range gateway.Spec.Listeners { + accepted := true + attachedRouteCount, err := getAttachedRoutesCountForListener(ctx, gatewayReconciler.client, gateway, string(listener.Name)) + if err != nil { + attachedRouteCount = 0 + } + listenerStatus := gwapiv1.ListenerStatus{ + Name: listener.Name, + SupportedKinds: []gwapiv1.RouteGroupKind{}, + Conditions: []metav1.Condition{}, + AttachedRoutes: attachedRouteCount, + } + + listenerDefinedAllowedKinds := listener.AllowedRoutes.Kinds + actualKinds := make([]gwapiv1.Kind, len(listenerDefinedAllowedKinds)) + for i, obj := range listenerDefinedAllowedKinds { + actualKinds[i] = obj.Kind + } + intersectionKinds := findIntersectionKinds(supportedKinds, actualKinds) + if len(intersectionKinds) == 0 { + // If listener does not define any supported kinds then we need to support all of the default supported kinds by the implementation + intersectionKinds = supportedKinds + } + for _, kind := range intersectionKinds { + listenerStatus.SupportedKinds = append(listenerStatus.SupportedKinds, gwapiv1.RouteGroupKind{ + Group: (*gwapiv1.Group)(&gwapiv1.GroupVersion.Group), + Kind: kind, + }) + } + + if len(intersectionKinds) < len(listenerDefinedAllowedKinds) { + accepted = false + listenerStatus.Conditions = append(listenerStatus.Conditions, metav1.Condition{ + Type: string(gwapiv1.ListenerConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: string(gwapiv1.ListenerReasonInvalidRouteKinds), + LastTransitionTime: metav1.Now(), + ObservedGeneration: gateway.Generation, + }) + } + if listener.Protocol == gwapiv1.HTTPProtocolType { continue } data, err := gatewayReconciler.resolveListenerSecretRefs(ctx, &listener.TLS.CertificateRefs[0], string(namespace)) if err != nil { + accepted = false + listenerStatus.Conditions = append(listenerStatus.Conditions, metav1.Condition{ + Type: string(gwapiv1.ListenerConditionResolvedRefs), + Status: metav1.ConditionFalse, + Reason: string(gwapiv1.ListenerReasonInvalidCertificateRef), + LastTransitionTime: metav1.Now(), + ObservedGeneration: gateway.Generation, + }) loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3105, logging.BLOCKER, "Error resolving listener certificates: %v", err)) - return nil, err + return nil, listenerstatuses, err } resolvedListenerCerts[string(listener.Name)] = data + if accepted { + listenerStatus.Conditions = append(listenerStatus.Conditions, metav1.Condition{ + Type: string(gwapiv1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(gwapiv1.ListenerReasonResolvedRefs), + LastTransitionTime: metav1.Now(), + ObservedGeneration: gateway.Generation, + }) + listenerStatus.Conditions = append(listenerStatus.Conditions, metav1.Condition{ + Type: string(gwapiv1.ListenerConditionProgrammed), + Status: metav1.ConditionTrue, + Reason: string(gwapiv1.ListenerReasonResolvedRefs), + LastTransitionTime: metav1.Now(), + ObservedGeneration: gateway.Generation, + }) + + } + listenerstatuses = append(listenerstatuses, listenerStatus) + loggers.LoggerAPKOperator.Debugf("A listener status is added for listener: %s", string(listenerStatus.Name)) } gatewayState.GatewayResolvedListenerCerts = resolvedListenerCerts if gatewayState.GatewayAPIPolicies, err = gatewayReconciler.getAPIPoliciesForGateway(ctx, &gateway); err != nil { - return nil, fmt.Errorf("error while getting gateway apipolicy for gateway: %s, %s", utils.NamespacedName(&gateway).String(), err.Error()) + return nil, listenerstatuses, fmt.Errorf("error while getting gateway apipolicy for gateway: %s, %s", utils.NamespacedName(&gateway).String(), err.Error()) } if gatewayState.GatewayInterceptorServiceMapping, err = gatewayReconciler.getInterceptorServicesForGateway(ctx, gatewayState.GatewayAPIPolicies); err != nil { - return nil, fmt.Errorf("error while getting interceptor service for gateway: %s, %s", utils.NamespacedName(&gateway).String(), err.Error()) + return nil, listenerstatuses, fmt.Errorf("error while getting interceptor service for gateway: %s, %s", utils.NamespacedName(&gateway).String(), err.Error()) } customRateLimitPolicies, err := gatewayReconciler.getCustomRateLimitPoliciesForGateway(utils.NamespacedName(&gateway)) if err != nil { @@ -247,7 +346,7 @@ func (gatewayReconciler *GatewayReconciler) resolveGatewayState(ctx context.Cont } gatewayState.GatewayCustomRateLimitPolicies = customRateLimitPolicies gatewayState.GatewayBackendMapping = gatewayReconciler.getResolvedBackendsMapping(ctx, gatewayState) - return gatewayState, nil + return gatewayState, listenerstatuses, nil } func (gatewayReconciler *GatewayReconciler) getAPIPoliciesForGateway(ctx context.Context, @@ -344,6 +443,34 @@ func (gatewayReconciler *GatewayReconciler) getGatewaysForBackend(ctx context.Co return requests } +// getHTTPRoutes returns the list of gateway reconcile requests +func (gatewayReconciler *GatewayReconciler) getHTTPRoutes(ctx context.Context, obj k8client.Object) []reconcile.Request { + httpRoute, ok := obj.(*gwapiv1.HTTPRoute) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3107, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", httpRoute)) + return []reconcile.Request{} + } + requests := []reconcile.Request{} + for _, refs := range httpRoute.Spec.ParentRefs { + if *refs.Kind == constants.KindGateway { + namespace := "" + if refs.Namespace != nil { + namespace = string(*refs.Namespace) + } + if namespace == "" { + namespace = httpRoute.Namespace + } + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: string(refs.Name), + }, + }) + } + } + return requests +} + // getAPIsForInterceptorService triggers the Gateway controller reconcile method based on the changes detected // in InterceptorService resources. func (gatewayReconciler *GatewayReconciler) getAPIsForInterceptorService(ctx context.Context, obj k8client.Object) []reconcile.Request { @@ -449,17 +576,14 @@ func (gatewayReconciler *GatewayReconciler) getGatewaysForConfigMap(ctx context. } // handleStatus updates the Gateway CR update -func (gatewayReconciler *GatewayReconciler) handleGatewayStatus(gatewayKey types.NamespacedName, state string, events []string) { - accept := false +func (gatewayReconciler *GatewayReconciler) handleGatewayStatus(gatewayKey types.NamespacedName, state string, + events []string, listeners []gwapiv1.ListenerStatus) { message := "" - //event := "" switch state { case constants.Create: - accept = true message = "Gateway is deployed successfully" case constants.Update: - accept = true message = fmt.Sprintf("Gateway update is deployed successfully. %v Updated", events) } timeNow := metav1.Now() @@ -475,18 +599,26 @@ func (gatewayReconciler *GatewayReconciler) handleGatewayStatus(gatewayKey types } hCopy := h.DeepCopy() var gwCondition []metav1.Condition = hCopy.Status.Conditions - gwCondition[0].Status = "Unknown" - if accept { - gwCondition[0].Status = "True" - } else { - gwCondition[0].Status = "False" - } + generation := hCopy.ObjectMeta.Generation + gwCondition[0].Status = "True" gwCondition[0].Message = message gwCondition[0].LastTransitionTime = timeNow // gwCondition[0].Reason = append(gwCondition[0].Reason, event) gwCondition[0].Reason = "Reconciled" - gwCondition[0].Type = state + gwCondition[0].Type = constants.Accept + for i := range gwCondition { + // Assign generation to ObservedGeneration + gwCondition[i].ObservedGeneration = generation + } hCopy.Status.Conditions = gwCondition + for _, listener := range hCopy.Status.Listeners { + for _, listener1 := range listeners { + if string(listener.Name) == string(listener1.Name) { + listener1.AttachedRoutes = listener.AttachedRoutes + } + } + } + hCopy.Status.Listeners = listeners return hCopy }, }) @@ -619,3 +751,92 @@ func addGatewayIndexes(ctx context.Context, mgr manager.Manager) error { }) return err } + +func findIntersectionKinds(list1, list2 []gwapiv1.Kind) []gwapiv1.Kind { + intersection := []gwapiv1.Kind{} + set := make(map[string]bool) + for _, v := range list1 { + set[string(v)] = true + } + for _, v := range list2 { + if set[string(v)] { + intersection = append(intersection, v) + } + } + return intersection +} + +// findDiffFromSecondListKinds return a list of elements in list2 that are not in the list1 +func findDiffFromSecondListKinds(list1, list2 []gwapiv1.Kind) []gwapiv1.Kind { + diff := []gwapiv1.Kind{} + set := make(map[string]bool) + for _, v := range list1 { + set[string(v)] = true + } + for _, v := range list2 { + if !set[string(v)] { + diff = append(diff, v) + } + } + return diff +} + +// getAttachedRoutesForListener returns the attached route count for a specific listener in a gatway +func getAttachedRoutesCountForListener(ctx context.Context, client k8client.Client, gateway gwapiv1.Gateway, listenerName string) (int32, error) { + httpRouteList := gwapiv1.HTTPRouteList{} + if err := client.List(ctx, &httpRouteList); err != nil { + return 0, err + } + + var attachedRoutesCount int32 + for _, httpRoute := range httpRouteList.Items { + _, found := common.FindElement(httpRoute.Status.Parents, func(parentStatus gwapiv1.RouteParentStatus) bool { + parentNamespacedName := types.NamespacedName{ + Namespace: string(*parentStatus.ParentRef.Namespace), + Name: string(parentStatus.ParentRef.Name), + }.String() + gatewayNamespacedName := utils.NamespacedName(&gateway).String() + if parentNamespacedName == gatewayNamespacedName { + if len(parentStatus.Conditions) >= 1 && parentStatus.Conditions[0].Status == metav1.ConditionTrue { + // Check whether the listername matches + _, matched := common.FindElement(httpRoute.Spec.ParentRefs, func(parentRef gwapiv1.ParentReference) bool { + if string(*parentRef.SectionName) == listenerName { + return true + } + return false + }) + return matched + } + } + return false + }) + if found { + attachedRoutesCount++ + } + } + return attachedRoutesCount, nil +} + +func isStatusChanged(gateway gwapiv1.Gateway, statuses []gwapiv1.ListenerStatus) bool { + if len(gateway.Status.Listeners) != len(statuses) { + return true + } + for _, status1 := range gateway.Status.Listeners { + flag := false + for _, status2 := range statuses { + if status1.Name == status2.Name && + status1.AttachedRoutes == status2.AttachedRoutes && + len(status1.Conditions) == len(status2.Conditions) && + len(status1.SupportedKinds) == len(status2.SupportedKinds) { + flag = common.BothListContainsSameConditions(status1.Conditions, status2.Conditions) + if flag { + continue + } + } + } + if !flag { + return true + } + } + return false +} diff --git a/adapter/internal/operator/controllers/dp/httproute_controller.go b/adapter/internal/operator/controllers/dp/httproute_controller.go new file mode 100644 index 0000000000..807a7a7d64 --- /dev/null +++ b/adapter/internal/operator/controllers/dp/httproute_controller.go @@ -0,0 +1,530 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dp + +import ( + "context" + "fmt" + + "github.com/wso2/apk/adapter/config" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" + "github.com/wso2/apk/adapter/internal/loggers" + "github.com/wso2/apk/adapter/internal/operator/constants" + "github.com/wso2/apk/adapter/internal/operator/status" + "github.com/wso2/apk/adapter/internal/operator/synchronizer" + "github.com/wso2/apk/adapter/internal/operator/utils" + "github.com/wso2/apk/adapter/pkg/logging" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + k8client "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +var httprouteParentKind = "Gateway" + +// HTTPRouteReconciler reconciles a HttpRoute object +type HTTPRouteReconciler struct { + client k8client.Client + ods *synchronizer.OperatorDataStore + statusUpdater *status.UpdateHandler + mgr manager.Manager +} + +const ( + conditionTypeProgrammed = "Programmed" + conditionReasonProgrammedUnknown gwapiv1.RouteConditionReason = "Unknown" + conditionReasonConfiguredInGateway gwapiv1.RouteConditionReason = "ConfiguredInGateway" + conditionReasonTranslationError gwapiv1.RouteConditionReason = "TranslationError" +) + +type referencedGatewaysAndCondition struct { + gateway *gwapiv1.Gateway + condition metav1.Condition + listenerName string +} + +// NewHTTPRouteController creates a new HttpRoute controller instance. Httproute Controllers watches for gwapiv1.HTTPRoute. +func NewHTTPRouteController(mgr manager.Manager, operatorDataStore *synchronizer.OperatorDataStore, statusUpdater *status.UpdateHandler) error { + r := &HTTPRouteReconciler{ + client: mgr.GetClient(), + ods: operatorDataStore, + statusUpdater: statusUpdater, + mgr: mgr, + } + c, err := controller.New(constants.HTTPRouteController, mgr, controller.Options{Reconciler: r}) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3119, logging.BLOCKER, "Error creating HttpRoute controller, error: %v", err)) + return err + } + conf := config.ReadConfigs() + predicates := []predicate.Predicate{predicate.NewPredicateFuncs(utils.FilterByNamespaces(conf.Adapter.Operator.Namespaces))} + + if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.HTTPRoute{}), &handler.EnqueueRequestForObject{}, + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3100, logging.BLOCKER, "Error watching HttpRoute resources: %v", err)) + return err + } + + if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.Gateway{}), + handler.EnqueueRequestsFromMapFunc(r.handleGateway), predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3121, logging.BLOCKER, "Error watching Gateway resources: %v", err)) + return err + } + + loggers.LoggerAPKOperator.Info("HttpRoute Controller successfully started. Watching Httproute Objects....") + return nil +} + +//+kubebuilder:rbac:groups=dp.wso2.com,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dp.wso2.com,resources=httproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dp.wso2.com,resources=httproutes/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the HttpRoute object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile +func (httpRouteReconciler *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Check whether the Httproute CR exist, if not consider as a DELETE event. + loggers.LoggerAPKOperator.Infof("Reconciling HttpRoute... %s", req.NamespacedName.String()) + var httproute gwapiv1.HTTPRoute + if err := httpRouteReconciler.client.Get(ctx, req.NamespacedName, &httproute); err != nil { + return ctrl.Result{}, nil + } + supportedGatways := getSupportedGatewaysForRoute(ctx, httpRouteReconciler.client, httproute) + existingStatuses := httproute.Status.Parents + // Check whether the route has supported gateways + if supportedGatways == nil || len(supportedGatways) == 0 { + loggers.LoggerAPKOperator.Debugf("Could not find any supported gateway for the httproute.") + + // There are no supported gateways found for this http route. We need to cleanup the parent statuses if needed + newStatuses := make([]gwapiv1.RouteParentStatus, 0) + for _, status := range existingStatuses { + if string(status.ControllerName) == GetControllerName() { + newStatuses = append(newStatuses, status) + } + } + if len(newStatuses) != len(existingStatuses) { + loggers.LoggerAPKOperator.Debugf("Cleaning up unnecessary statuses from HTTPRoute cr.") + // Need to change the statuses + httpRouteReconciler.statusUpdater.Send(status.Update{ + NamespacedName: req.NamespacedName, + Resource: new(gwapiv1.HTTPRoute), + UpdateStatus: func(obj k8client.Object) k8client.Object { + h, ok := obj.(*gwapiv1.HTTPRoute) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3109, logging.BLOCKER, "Error while updating HttpRoute status %v", obj)) + } + hCopy := h.DeepCopy() + hCopy.Status.Parents = newStatuses + return hCopy + }, + }) + } + // There is no status change needed + } else { + // We found some supported gateways so we need to update parent statuses + // Create a hashMap to store the already existing statuses by parentref properties as key so that we will not duplicate the status. + routeParentStatusMap := make(map[string]gwapiv1.RouteParentStatus) + for _, routeParentStatus := range existingStatuses { + key := fmt.Sprintf("%s/%s/%s/%s", *routeParentStatus.ParentRef.Group, *routeParentStatus.ParentRef.Kind, routeParentStatus.ParentRef.Name, *routeParentStatus.ParentRef.Namespace) + routeParentStatusMap[key] = routeParentStatus + } + needsStatusUpdate := false + // Prepare a local copy of status for supported gateways + for _, gatewayWithCondition := range supportedGatways { + gatewayParentStatus := &gwapiv1.RouteParentStatus{ + ParentRef: gwapiv1.ParentReference{ + Group: (*gwapiv1.Group)(&gwapiv1.GroupVersion.Group), + Kind: common.PointerCopy(gwapiv1.Kind(httprouteParentKind)), + Namespace: (*gwapiv1.Namespace)(&gatewayWithCondition.gateway.Namespace), + Name: gwapiv1.ObjectName(gatewayWithCondition.gateway.Name), + }, + ControllerName: gwapiv1.GatewayController(GetControllerName()), + Conditions: []metav1.Condition{{ + Type: gatewayWithCondition.condition.Type, + Status: gatewayWithCondition.condition.Status, + ObservedGeneration: httproute.Generation, + LastTransitionTime: metav1.Now(), + Reason: gatewayWithCondition.condition.Reason, + }}, + } + key := fmt.Sprintf("%s/%s/%s/%s", *gatewayParentStatus.ParentRef.Group, *gatewayParentStatus.ParentRef.Kind, gatewayParentStatus.ParentRef.Name, *gatewayParentStatus.ParentRef.Namespace) + foundStatus, exists := routeParentStatusMap[key] + statusChanged := true + if exists { + if foundStatus.ControllerName == gatewayParentStatus.ControllerName && + *foundStatus.ParentRef.Kind == *gatewayParentStatus.ParentRef.Kind && + foundStatus.ParentRef.Name == gatewayParentStatus.ParentRef.Name && + *foundStatus.ParentRef.Namespace == *gatewayParentStatus.ParentRef.Namespace && + *foundStatus.ParentRef.Group == *gatewayParentStatus.ParentRef.Group && + len(foundStatus.Conditions) > 0 && + common.AreConditionsSame(foundStatus.Conditions[0], gatewayParentStatus.Conditions[0]) { + statusChanged = false + } + } + needsStatusUpdate = needsStatusUpdate || statusChanged + if statusChanged { + status, found := common.FindElement(routeParentStatusMap[key].Conditions, func(cond metav1.Condition) bool { + if cond.Type == conditionTypeProgrammed { + return true + } + return false + }) + if !found { + gatewayParentStatus.Conditions = append(gatewayParentStatus.Conditions, metav1.Condition{ + Type: conditionTypeProgrammed, + Status: metav1.ConditionUnknown, + Reason: string(conditionReasonProgrammedUnknown), + ObservedGeneration: httproute.Generation, + LastTransitionTime: metav1.Now(), + }) + } else { + gatewayParentStatus.Conditions = append(gatewayParentStatus.Conditions, status) + } + routeParentStatusMap[key] = *gatewayParentStatus + } + } + + if needsStatusUpdate || len(httproute.Status.Parents) != len(routeParentStatusMap) { + httproute.Status.Parents = make([]gwapiv1.RouteParentStatus, 0) + for _, parentStatus := range routeParentStatusMap { + httproute.Status.Parents = append(httproute.Status.Parents, parentStatus) + } + + httpRouteReconciler.statusUpdater.Send(status.Update{ + NamespacedName: req.NamespacedName, + Resource: new(gwapiv1.HTTPRoute), + UpdateStatus: func(obj k8client.Object) k8client.Object { + h, ok := obj.(*gwapiv1.HTTPRoute) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3109, logging.BLOCKER, "Error while updating HttpRoute status %v", obj)) + } + hCopy := h.DeepCopy() + hCopy.Status.Parents = httproute.Status.Parents + return hCopy + }, + }) + } + + } + + return ctrl.Result{}, nil +} + +// handleGateway handles the gateway changes and create reconcile reuqest for HTTPRoute reconciler +func (httpRouteReconciler *HTTPRouteReconciler) handleGateway(ctx context.Context, obj k8client.Object) []reconcile.Request { + gateway, ok := obj.(*gwapiv1.Gateway) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", gateway)) + return []reconcile.Request{} + } + + httpRouteList := &gwapiv1.HTTPRouteList{} + if err := httpRouteReconciler.client.List(ctx, httpRouteList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(gatewayHTTPRouteIndex, utils.NamespacedName(gateway).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2625, logging.CRITICAL, "Unable to find associated HTTPRoutes: %s", utils.NamespacedName(gateway).String())) + return []reconcile.Request{} + } + + if len(httpRouteList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("HTTPRoutes for Gateway not found: %s", utils.NamespacedName(gateway).String()) + return []reconcile.Request{} + } + + requests := []reconcile.Request{} + for item := range httpRouteList.Items { + httpRoute := httpRouteList.Items[item] + if refersGateway(httpRoute, *gateway) { + requests = append(requests, reconcile.Request{ + NamespacedName: utils.NamespacedName(&httpRoute), + }) + } else { + loggers.LoggerAPKOperator.Debugf("Gateway change observed but HttpRoute: %s does not belongs to this gateway: %s hence not reconciling.", + utils.NamespacedName(&httpRoute).String(), + utils.NamespacedName(gateway).String()) + } + } + return requests +} + +func getSupportedGatewaysForRoute(ctx context.Context, client k8client.Client, httpRoute gwapiv1.HTTPRoute) []referencedGatewaysAndCondition { + parentRefs := httpRoute.Spec.ParentRefs + if parentRefs == nil { + loggers.LoggerAPKOperator.Errorf("Parent ref not found for HTTPRoute: %s/%s", httpRoute.Namespace, httpRoute.Name) + return nil + } + referencedGatewayList := make([]referencedGatewaysAndCondition, 0) + for _, parentRef := range parentRefs { + namespace := httpRoute.GetNamespace() + if parentRef.Namespace != nil { + namespace = string(*parentRef.Namespace) + } + name := string(parentRef.Name) + + // Try to fetch the referenced gateway + gateway := gwapiv1.Gateway{} + if err := client.Get(ctx, types.NamespacedName{ + Namespace: namespace, + Name: name, + }, &gateway); err != nil { + if apierrors.IsNotFound(err) { + loggers.LoggerAPKOperator.Infof("Could not find gateway: %s/%s", namespace, name) + // There might be other gateway in this list that can be found. So keep search. + continue + } + loggers.LoggerAPKOperator.Errorf("Error while fetching gateway: %s/%s. Error: %s", namespace, name, err) + return nil + } + var ( + httpRouteMatched = false + hostnameMatched = false + portMatched = false + satisfiesAllowedRoutes = false + satisfiesSupportedKinds = false + satisfiesListenerName = false + ) + + for _, listener := range gateway.Spec.Listeners { + if ok, err := checkIfRouteMatchesAllowedRoutes(ctx, client, httpRoute, listener, gateway.Namespace, parentRef.Namespace); err != nil { + return nil + } else if !ok { + loggers.LoggerAPKOperator.Debugf("HttpRoute(%s/%s) did not match any allowed routes in gateway: %s/%s, listener: %s", httpRoute.Namespace, httpRoute.Name, namespace, name, listener.Name) + continue + } + satisfiesAllowedRoutes = true + if err := checkIfMatchingReadyListenerExistsInStatus(httpRoute, listener, gateway.Status.Listeners); err != nil { + loggers.LoggerAPKOperator.Debugf("Gateway(%s/%s) listener: %s does not have a ready listener for this HttpRoute(%s/%s). Error: %s", namespace, name, listener.Name, httpRoute.Namespace, httpRoute.Name, err.Error()) + continue + } + satisfiesSupportedKinds = true + if parentRef.SectionName != nil { + if *parentRef.SectionName != "" && *parentRef.SectionName != listener.Name { + loggers.LoggerAPKOperator.Debugf("Gateway(%s/%s) listener: %s does not have a matching listener with section name %s for this HttpRoute(%s/%s)", namespace, name, listener.Name, *parentRef.SectionName, httpRoute.Namespace, httpRoute.Name) + continue + } + satisfiesListenerName = true + } + if parentRef.Port != nil { + if *parentRef.Port != listener.Port { + loggers.LoggerAPKOperator.Debugf("Gateway(%s/%s) listener: %s does not have a matching port (%d) for this HttpRoute(%s/%s)", namespace, name, listener.Name, int32(*parentRef.Port), httpRoute.Namespace, httpRoute.Name) + continue + } + portMatched = true + } + hostnameMatched = isGatewayListenerHostnameMatched(listener, httpRoute.Spec.Hostnames) + httpRouteMatched = hostnameMatched + } + + if httpRouteMatched { + var listenerName string + if parentRef.SectionName != nil && *parentRef.SectionName != "" { + listenerName = string(*parentRef.SectionName) + } + + referencedGatewayList = append(referencedGatewayList, referencedGatewaysAndCondition{ + gateway: &gateway, + listenerName: listenerName, + condition: metav1.Condition{ + Type: string(gwapiv1.RouteConditionAccepted), + Status: metav1.ConditionTrue, + Reason: string(gwapiv1.RouteReasonAccepted), + ObservedGeneration: httpRoute.GetGeneration(), + }, + }) + } else { + reason := gwapiv1.RouteReasonNoMatchingParent + if !hostnameMatched { + reason = gwapiv1.RouteReasonNoMatchingListenerHostname + } else if (parentRef.SectionName) != nil && !satisfiesListenerName { + reason = gwapiv1.RouteReasonNoMatchingParent + } else if (parentRef.Port != nil) && !portMatched { + reason = gwapiv1.RouteReasonNoMatchingParent + } else if !satisfiesAllowedRoutes || !satisfiesSupportedKinds { + reason = gwapiv1.RouteReasonNotAllowedByListeners + } + var listenerName string + if parentRef.SectionName != nil && *parentRef.SectionName != "" { + listenerName = string(*parentRef.SectionName) + } + + referencedGatewayList = append(referencedGatewayList, referencedGatewaysAndCondition{ + gateway: &gateway, + listenerName: listenerName, + condition: metav1.Condition{ + Type: string(gwapiv1.RouteConditionAccepted), + Status: metav1.ConditionFalse, + Reason: string(reason), + ObservedGeneration: httpRoute.GetGeneration(), + }, + }) + } + } + return referencedGatewayList +} + +// checkIfRouteMatchesAllowedRoutes determines if the provided route matches the +// criteria defined in the listener's AllowedRoutes field. +func checkIfRouteMatchesAllowedRoutes( + ctx context.Context, + client k8client.Client, + httpRoute gwapiv1.HTTPRoute, + listener gwapiv1.Listener, + gatewayNamespace string, + parentRefNamespace *gwapiv1.Namespace, +) (bool, error) { + if listener.AllowedRoutes == nil { + return true, nil + } + + if len(listener.AllowedRoutes.Kinds) > 0 { + _, found := common.FindElement(listener.AllowedRoutes.Kinds, func(allowedGroupKind gwapiv1.RouteGroupKind) bool { + httpRouteVersionKind := httpRoute.GetObjectKind().GroupVersionKind() + return (allowedGroupKind.Group != nil && string(*allowedGroupKind.Group) == httpRouteVersionKind.Group) && string(allowedGroupKind.Kind) == httpRouteVersionKind.Kind + }) + if !found { + return false, nil + } + } + + if listener.AllowedRoutes.Namespaces == nil || listener.AllowedRoutes.Namespaces.From == nil { + return true, nil + } + + switch *listener.AllowedRoutes.Namespaces.From { + case gwapiv1.NamespacesFromAll: + return true, nil + + case gwapiv1.NamespacesFromSame: + if parentRefNamespace == nil { + return gatewayNamespace == httpRoute.GetNamespace(), nil + } + return httpRoute.GetNamespace() == string(*parentRefNamespace), nil + + case gwapiv1.NamespacesFromSelector: + namespace := corev1.Namespace{} + if err := client.Get(ctx, types.NamespacedName{Name: httpRoute.GetNamespace()}, &namespace); err != nil { + return false, fmt.Errorf("failed to get namespace %s: %w", httpRoute.GetNamespace(), err) + } + + s, err := metav1.LabelSelectorAsSelector(listener.AllowedRoutes.Namespaces.Selector) + if err != nil { + return false, fmt.Errorf( + "failed to convert AllowedRoutes LabelSelector %s to Selector for listener %s: %w", + listener.AllowedRoutes.Namespaces.Selector, listener.Name, err, + ) + } + + ok := s.Matches(labels.Set(namespace.Labels)) + return ok, nil + + default: + return false, fmt.Errorf( + "unknown listener.AllowedRoutes.Namespaces.From value: %s for listener %s", + *listener.AllowedRoutes.Namespaces.From, listener.Name, + ) + } +} + +// checkIfMatchingReadyListenerExistsInStatus determines if there exists a matching ready listener +// in the provided list of listener statuses. +func checkIfMatchingReadyListenerExistsInStatus(route gwapiv1.HTTPRoute, listener gwapiv1.Listener, listenerStatuses []gwapiv1.ListenerStatus) error { + + // Find the relative gateway listener status for the gateway listener + listenerStatus, listenerFound := common.FindElement(listenerStatuses, func(listenerStatus gwapiv1.ListenerStatus) bool { + return listenerStatus.Name != listener.Name + }) + if !listenerFound { + return fmt.Errorf("Cannot find a listener status for this route in gateway") + } + + // Check if the programmed status exists + programmedStatus, foundProgrammedStatus := common.FindElement(listenerStatus.Conditions, func(c metav1.Condition) bool { + return c.Type == string(gwapiv1.ListenerConditionProgrammed) + }) + if !foundProgrammedStatus { + return fmt.Errorf("Cannot find a programmed listener status for this route in gateway") + } + if programmedStatus.Status != "True" { + return fmt.Errorf("Programmed status is not active yet") + } + + return nil +} + +func isGatewayListenerHostnameMatched(listener gwapiv1.Listener, hostnames []gwapiv1.Hostname) bool { + // If httpRoute does not specify any hostnames then we can assume it accept gateway listener's hostname + if len(hostnames) == 0 { + return true + } + + // If listener hostname is nil or empty it will accept all hostnames from the httpRoute + if listener.Hostname == nil || *listener.Hostname == "" { + return true + } + + for _, hostname := range hostnames { + if common.MatchesHostname(string(hostname), string(*listener.Hostname)) { + return true + } + } + + return false +} + +func refersGateway(httpRoute gwapiv1.HTTPRoute, gateway gwapiv1.Gateway) bool { + _, found := common.FindElement(httpRoute.Spec.ParentRefs, func(parentRef gwapiv1.ParentReference) bool { + namespace := "" + if parentRef.Namespace != nil { + namespace = string(*parentRef.Namespace) + } + if namespace == "" { + namespace = httpRoute.GetNamespace() + } + referingGatewayNamespacedName := types.NamespacedName{ + Namespace: namespace, + Name: string(parentRef.Name), + } + gatewayNamespace := types.NamespacedName{ + Namespace: gateway.Namespace, + Name: gateway.Name, + } + if referingGatewayNamespacedName.String() == gatewayNamespace.String() { + return true + } + + return false + }) + return found +} diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index 87cf2bf916..6fd4c6f340 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -139,6 +139,9 @@ func InitOperator(metricsConfig config.Metrics) { if err := dpcontrollers.NewAPIController(mgr, operatorDataStore, updateHandler, &ch, &successChannel); err != nil { loggers.LoggerAPKOperator.Errorf("Error creating API controller: %v", err) } + if err := dpcontrollers.NewHTTPRouteController(mgr, operatorDataStore, updateHandler); err != nil { + loggers.LoggerAPKOperator.Errorf("Error creating HTTPRouteadapter/internal/operator/synchronizer/gateway_state.go controller: %v", err) + } if err := dpcontrollers.NewTokenIssuerReconciler(mgr); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3114, logging.BLOCKER, "Error creating JWT Issuer controller: %v", err)) diff --git a/adapter/internal/operator/synchronizer/gateway_state.go b/adapter/internal/operator/synchronizer/gateway_state.go index 947685d3d7..6566311d56 100644 --- a/adapter/internal/operator/synchronizer/gateway_state.go +++ b/adapter/internal/operator/synchronizer/gateway_state.go @@ -39,4 +39,5 @@ type GatewayStateData struct { GatewayBackendMapping map[string]*v1alpha1.ResolvedBackend GatewayInterceptorServiceMapping map[string]v1alpha1.InterceptorService GatewayCustomRateLimitPolicies map[string]*v1alpha1.RateLimitPolicy + GatewayHTTPRoutes map[string]*gwapiv1.HTTPRoute } diff --git a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml index f034acbe53..ece4f3ec2e 100644 --- a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml +++ b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml @@ -246,7 +246,7 @@ data: logLevel = "{{ .Values.wso2.apk.dp.adapter.logging.level | default "INFO" }}" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" LogFormat = "{{ .Values.wso2.apk.dp.adapter.logging.logFormat | default "TEXT" }}" # Values can be "JSON", "TEXT" {{- else }} - logLevel = "INFO" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" + logLevel = "DEBG" # LogLevels can be "DEBG", "FATL", "ERRO", "WARN", "INFO", "PANC" LogFormat = "TEXT" # Values can be "JSON", "TEXT" {{- end }} diff --git a/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal b/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal index 38ff5974c8..1b4fcd3f79 100644 --- a/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal +++ b/runtime/config-deployer-service/ballerina/modules/model/HttpRoute.bal @@ -132,6 +132,7 @@ public type HTTPRoute record {| string kind = "HTTPRoute"; Metadata metadata; HTTPRouteSpec spec; + anydata status = (); |}; public type ParentReference record {| diff --git a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/BaseSteps.java b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/BaseSteps.java index de5b55014e..0e4a4be252 100644 --- a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/BaseSteps.java +++ b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/BaseSteps.java @@ -248,7 +248,7 @@ public void waitForNextMinuteStrictly() throws InterruptedException { @Then("I wait for {int} minute") public void waitForMinute(int minute) throws InterruptedException { - Thread.sleep(minute * 1000); + Thread.sleep(minute * 1000 * 60); } @Then("I wait for {int} seconds") @@ -256,6 +256,11 @@ public void waitForSeconds(int seconds) throws InterruptedException { Thread.sleep(seconds * 1000); } + @Then("I wait for api deployment") + public void waitAPIDeployment() throws InterruptedException { + waitForSeconds(30); + } + @Then("the response headers contains key {string} and value {string}") public void containsHeader(String key, String value) { key = Utils.resolveVariables(key, sharedContext.getValueStore()); diff --git a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature index b96dff9607..0adeb6fca4 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature @@ -44,7 +44,7 @@ Feature: API Definition Endpoint And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 - And I wait for 1 minute + And I wait for api deployment Then I set headers | Authorization | bearer ${accessToken} | And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" @@ -63,7 +63,7 @@ Feature: API Definition Endpoint And the definition file "artifacts/definitions/employees_api.json" And make the API deployment request Then the response status code should be 200 - And I wait for 1 minute + And I wait for api deployment Then I set headers | Authorization | bearer ${accessToken} | And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" diff --git a/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature index 41bff9d905..4c75886cda 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature @@ -7,7 +7,7 @@ Feature: API Deployment with Global Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "579ba27a1e03e2fdf099d1b6745e265f2d495606" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/globalinterceptor/1.0.0/get" with body "" diff --git a/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature index 27e5a37fb6..14af1c5021 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature @@ -7,7 +7,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -18,7 +18,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -29,7 +29,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -41,7 +41,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -53,7 +53,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers | Authorization | bearer ${accessToken} | And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -65,7 +65,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers | Authorization | bearer ${accessToken} | And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -77,7 +77,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers | Authorization | bearer ${accessToken} | And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -89,7 +89,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" @@ -101,7 +101,7 @@ Feature: API Deployment with Interceptor And make the API deployment request Then the response status code should be 200 And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute + And I wait for api deployment Then I set headers |Authorization|bearer ${accessToken}| And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body ""