From b81de35fa614d7d1a6b2bc9b8194932bd8af840d Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Mon, 13 Nov 2023 10:54:39 +0530 Subject: [PATCH] Add http, https and multiple port support --- adapter/internal/data_holder/data-holder.go | 91 ++++ .../internal/discovery/xds/common/utils.go | 41 +- adapter/internal/discovery/xds/server.go | 89 +++- adapter/internal/discovery/xds/server_test.go | 2 +- .../internal/oasparser/config_generator.go | 18 +- .../internal/oasparser/envoyconf/constants.go | 4 - .../internal/oasparser/envoyconf/listener.go | 445 +++++++++--------- .../oasparser/envoyconf/listener_test.go | 2 +- .../envoyconf/routes_with_clusters_test.go | 19 + .../controllers/dp/gateway_controller.go | 3 + adapter/internal/operator/operator.go | 2 +- .../operator/synchronizer/data_store.go | 12 +- .../synchronizer/gateway_synchronizer.go | 3 + .../operator/synchronizer/synchronizer.go | 54 ++- helm-charts/README.md | 3 + .../config-deployer/config-api-route.yaml | 2 +- .../config-deploy-api-route.yaml | 2 +- .../gateway-components/adapter/gateway.yaml | 8 +- .../gateway-runtime-deployment.yaml | 4 + .../gateway-runtime/gateway-service.yaml | 11 +- ...ticationEndpoint-domain-api-httproute.yaml | 2 +- .../idp/commonoauth-domain-api-httproute.yaml | 2 +- .../idp/dcr-domain-api-httproute.yaml | 2 +- .../idp/oauth-domain-api-httproute.yaml | 2 +- helm-charts/values.yaml.template | 7 + .../integration/api/APKGenerationSteps.java | 4 +- .../tests/api/APIDefinitionEndpoint.feature | 2 +- .../deploy-api-to-multiple-ports-listener.go | 78 +++ .../tests/multiple_port_listener.yaml | 88 ++++ .../integration/utils/http/http.go | 13 + 30 files changed, 745 insertions(+), 270 deletions(-) create mode 100644 adapter/internal/data_holder/data-holder.go create mode 100644 test/integration/integration/tests/deploy-api-to-multiple-ports-listener.go create mode 100644 test/integration/integration/tests/resources/tests/multiple_port_listener.yaml diff --git a/adapter/internal/data_holder/data-holder.go b/adapter/internal/data_holder/data-holder.go new file mode 100644 index 000000000..aa395d12a --- /dev/null +++ b/adapter/internal/data_holder/data-holder.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package data_holder + +import ( + k8types "k8s.io/apimachinery/pkg/types" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// The following variables will be used to store the state of the apk. +// This data should not be utilized by operator thread as its not designed for parallel access. +var ( + // This variable in the structure of gateway's namespaced name -> gateway + gatewayMap map[string]gwapiv1b1.Gateway +) + +func init() { + gatewayMap = make(map[string]gwapiv1b1.Gateway) +} + +// GetGatewayMap returns list of cached gateways +func GetGatewayMap() map[string]gwapiv1b1.Gateway { + return gatewayMap +} + +// UpdateGateway caches the gateway +func UpdateGateway(gateway gwapiv1b1.Gateway) { + gatewayMap[k8types.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}.String()] = gateway +} + +// RemoveGateway removes the gateway from the cache +func RemoveGateway(gateway gwapiv1b1.Gateway) { + delete(gatewayMap, k8types.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}.String()) +} + +// GetAllGatewayListeners return the list of all the listeners that stored in the gateway cache +func GetAllGatewayListeners() []gwapiv1b1.Listener { + listeners := make([]gwapiv1b1.Listener, 0) + for _, gateway := range gatewayMap { + for _, listener := range gateway.Spec.Listeners { + listeners = append(listeners, listener) + } + } + return listeners +} + + +// GetListenersAsPortalPortMap returns a map that have a structure protocol -> port -> list of listeners for that port and protocol combination +// Data is derived based on the current status of the gatwayMap cache +func GetListenersAsPortalPortMap() map[string]map[uint32][]gwapiv1b1.Listener{ + listenersAsPortalPortMap := make(map[string]map[uint32][]gwapiv1b1.Listener) + for _, gateway := range gatewayMap { + for _, listener := range gateway.Spec.Listeners { + protocol := string(listener.Protocol) + port := uint32(listener.Port) + if portMap, portFound := listenersAsPortalPortMap[protocol]; portFound { + if listenersList, listenerListFound := portMap[port]; listenerListFound { + if (listenersList == nil) { + listenersList = []gwapiv1b1.Listener{listener} + } else { + listenersList = append(listenersList, listener) + } + listenersAsPortalPortMap[protocol][port] = listenersList + } else { + listenerList := []gwapiv1b1.Listener{listener} + listenersAsPortalPortMap[protocol][port] = listenerList + } + } else { + listenersAsPortalPortMap[protocol] = make(map[uint32][]gwapiv1b1.Listener) + listenerList := []gwapiv1b1.Listener{listener} + listenersAsPortalPortMap[protocol][port] = listenerList + } + } + } + return listenersAsPortalPortMap +} diff --git a/adapter/internal/discovery/xds/common/utils.go b/adapter/internal/discovery/xds/common/utils.go index 99510fef2..b09cdb411 100644 --- a/adapter/internal/discovery/xds/common/utils.go +++ b/adapter/internal/discovery/xds/common/utils.go @@ -20,7 +20,9 @@ package common import ( "sync" - + "fmt" + "regexp" + "strings" discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" ) @@ -92,3 +94,40 @@ func GetNodeIdentifier(request *discovery.DiscoveryRequest) string { } return nodeIdentifier } + +// GetEnvoyListenerName prepares the envoy listener name based on the protocol and port +func GetEnvoyListenerName(protocol string, port uint32) string { + return fmt.Sprintf("%s_%d_listener", protocol, port) +} + +// GetEnvoyRouteConfigName prepares Envoy route config name based on Gateway spec's listener name and section name +func GetEnvoyRouteConfigName(listenerName string, sectionName string) string { + return fmt.Sprintf("%s_%s", listenerName, sectionName) +} + +// FindElement searches for an element in a slice based on a given predicate. +// It returns the element and true if the element was found. +func FindElement[T any](collection []T, predicate func(item T) bool) (T, bool) { + for _, item := range collection { + if predicate(item) { + return item, true + } + } + var dummy T + return dummy, false +} + +// MatchesHostname check whether the domain matches the hostname pattern +func MatchesHostname(domain, pattern string) bool { + // Escape special characters in the pattern and replace wildcard with regex pattern + pattern = strings.ReplaceAll(regexp.QuoteMeta(pattern), `\*`, `.*`) + // Append start and end of line anchors + pattern = "^" + pattern + "$" + + matched, err := regexp.MatchString(pattern, domain) + if err != nil { + return false + } + + return matched +} diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index dafea55f5..eea1dec46 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -40,6 +40,7 @@ import ( envoy_resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/wso2/apk/adapter/config" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" logger "github.com/wso2/apk/adapter/internal/loggers" logging "github.com/wso2/apk/adapter/internal/logging" oasParser "github.com/wso2/apk/adapter/internal/oasparser" @@ -53,6 +54,7 @@ import ( eventhubTypes "github.com/wso2/apk/adapter/pkg/eventhub/types" "github.com/wso2/apk/adapter/pkg/utils/stringutils" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "github.com/wso2/apk/adapter/internal/data_holder" ) // EnvoyInternalAPI struct use to hold envoy resources and adapter internal resources @@ -67,8 +69,8 @@ type EnvoyInternalAPI struct { // EnvoyGatewayConfig struct use to hold envoy gateway resources type EnvoyGatewayConfig struct { - listener *listenerv3.Listener - routeConfig *routev3.RouteConfiguration + listeners []*listenerv3.Listener + routeConfigs []*routev3.RouteConfiguration clusters []*clusterv3.Cluster endpoints []*corev3.Address customRateLimitPolicies []*model.CustomRateLimitPolicy @@ -125,6 +127,10 @@ const ( apiController string = "APIController" ) +type envoyRoutesWithSectionName struct { + routes []*routev3.Route +} + func maxRandomBigInt() *big.Int { return big.NewInt(int64(maxRandomInt)) } @@ -399,23 +405,61 @@ func GenerateEnvoyResoucesForGateway(gatewayName string) ([]types.Resource, } envoyGatewayConfig, gwFound := gatewayLabelConfigMap[gatewayName] - listener := envoyGatewayConfig.listener - if !gwFound || listener == nil { + listeners := envoyGatewayConfig.listeners + if !gwFound || listeners == nil || len(listeners) == 0 { return nil, nil, nil, nil, nil } - routesFromListener := listenerToRouteArrayMap[listener.Name] - var vhostToRouteArrayFilteredMap = make(map[string][]*routev3.Route) - for vhost, routes := range vhostToRouteArrayMap { - if vhost == systemHost || checkRoutes(routes, routesFromListener) { - vhostToRouteArrayFilteredMap[vhost] = routes + routeConfigs := make([]*routev3.RouteConfiguration, 0) + for _, listener := range listeners { + for vhost, routes := range vhostToRouteArrayMap { + matchedListener, found := common.FindElement(data_holder.GetAllGatewayListeners(), func(listenerLocal gwapiv1b1.Listener) bool { + if (listenerLocal.Hostname != nil && common.MatchesHostname(vhost, string(*listenerLocal.Hostname))) { + if (listener.Name == common.GetEnvoyListenerName(string(listenerLocal.Protocol), uint32(listenerLocal.Port))) { + return true + } + } + return false + }) + if found { + // Prepare the route config name based on the gateway listener section name. + routeConfigName := common.GetEnvoyRouteConfigName(listener.Name, string(matchedListener.Name)); + routesConfig := oasParser.GetRouteConfigs(map[string][]*routev3.Route{vhost:routes}, routeConfigName, envoyGatewayConfig.customRateLimitPolicies) + + routeConfigMatched, alreadyExistsInRouteConfigList := common.FindElement(routeConfigs, func(routeConf *routev3.RouteConfiguration) bool { + if (routeConf.Name == routesConfig.Name) { + return true + } + return false + }) + if alreadyExistsInRouteConfigList { + logger.LoggerAPKOperator.Debugf("Route already exists. %+v", routesConfig.Name) + routeConfigMatched.VirtualHosts = append(routeConfigMatched.VirtualHosts, routesConfig.VirtualHosts...) + } else { + routeConfigs = append(routeConfigs, routesConfig) + } + } else { + logger.LoggerAPKOperator.Errorf("Failed to find a matching gateway listener for this vhost: %s", vhost) + } + } + } + + // Find gateway listeners that has $systemHost as its hostname and add the system routeConfig referencing those listeners + gatewayListeners := data_holder.GetAllGatewayListeners() + for _, listener := range gatewayListeners { + if (systemHost == string(*listener.Hostname)) { + var vhostToRouteArrayFilteredMapForSystemEndpoints = make(map[string][]*routev3.Route) + vhostToRouteArrayFilteredMapForSystemEndpoints[systemHost] = vhostToRouteArrayMap[systemHost] + routeConfigName := common.GetEnvoyRouteConfigName(common.GetEnvoyListenerName(string(listener.Protocol), uint32(listener.Port)), string(listener.Name)) + systemRoutesConfig := oasParser.GetRouteConfigs(vhostToRouteArrayFilteredMapForSystemEndpoints, routeConfigName, envoyGatewayConfig.customRateLimitPolicies) + routeConfigs = append(routeConfigs, systemRoutesConfig) } } - routesConfig := oasParser.GetRouteConfigs(vhostToRouteArrayFilteredMap, listener.Name, envoyGatewayConfig.customRateLimitPolicies) - envoyGatewayConfig.routeConfig = routesConfig + + envoyGatewayConfig.routeConfigs = routeConfigs clusterArray = append(clusterArray, envoyGatewayConfig.clusters...) endpointArray = append(endpointArray, envoyGatewayConfig.endpoints...) - endpoints, clusters, listeners, routeConfigs := oasParser.GetCacheResources(endpointArray, clusterArray, listener, routesConfig) - return endpoints, clusters, listeners, routeConfigs, apis + listeners_, clusters, routeConfigs_, endpoints := oasParser.GetCacheResources(endpointArray, clusterArray, listeners, routeConfigs) + return listeners_, clusters, routeConfigs_, endpoints, apis } // function to check routes []*routev3.Route equlas routes []*routev3.Route @@ -488,7 +532,6 @@ func updateXdsCache(label string, endpoints []types.Resource, clusters []types.R logger.LoggerXds.ErrorC(logging.PrintError(logging.Error1414, logging.MAJOR, "Error while setting the snapshot : %v", errSetSnap.Error())) return false } - logger.LoggerXds.Infof("New Router cache updated for the label: " + label + " version: " + fmt.Sprint(version)) return true } @@ -641,7 +684,7 @@ func RemoveAPIFromOrgAPIMap(uuid string, orgID string) { } // UpdateAPICache updates the xDS cache related to the API Lifecycle event. -func UpdateAPICache(vHosts []string, newLabels []string, newlistenersForRoutes []string, adapterInternalAPI model.AdapterInternalAPI) error { +func UpdateAPICache(vHosts []string, newLabels []string, listener string, sectionName string, adapterInternalAPI model.AdapterInternalAPI) error { mutexForInternalMapUpdate.Lock() defer mutexForInternalMapUpdate.Unlock() @@ -671,6 +714,7 @@ func UpdateAPICache(vHosts []string, newLabels []string, newlistenersForRoutes [ // Create internal mappigs for new vHosts for _, vHost := range vHosts { + logger.LoggerAPKOperator.Debugf("Creating internal mapping for vhost: %s", vHost) apiUUID := adapterInternalAPI.UUID apiIdentifier := GenerateIdentifierForAPIWithUUID(vHost, apiUUID) var oldLabels []string @@ -700,10 +744,15 @@ func UpdateAPICache(vHosts []string, newLabels []string, newlistenersForRoutes [ endpointAddresses: endpoints, enforcerAPI: oasParser.GetEnforcerAPI(adapterInternalAPI, vHost), } - if _, ok := listenerToRouteArrayMap[newlistenersForRoutes[0]]; ok { - listenerToRouteArrayMap[newlistenersForRoutes[0]] = append(listenerToRouteArrayMap[newlistenersForRoutes[0]], routes...) + if _, ok := listenerToRouteArrayMap[listener]; ok { + routesList := listenerToRouteArrayMap[listener] + if (routesList == nil) { + routesList = make([]*routev3.Route, 0) + } + routesList = append(routesList, routes...) + listenerToRouteArrayMap[listener] = routesList } else { - listenerToRouteArrayMap[newlistenersForRoutes[0]] = routes + listenerToRouteArrayMap[listener] = routes } revisionStatus := updateXdsCacheOnAPIChange(oldLabels, newLabels) @@ -715,8 +764,8 @@ func UpdateAPICache(vHosts []string, newLabels []string, newlistenersForRoutes [ // UpdateGatewayCache updates the xDS cache related to the Gateway Lifecycle event. func UpdateGatewayCache(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map[string]map[string][]byte, gwLuaScript string, customRateLimitPolicies []*model.CustomRateLimitPolicy) error { - listener := oasParser.GetProductionListener(gateway, resolvedListenerCerts, gwLuaScript) - gatewayLabelConfigMap[gateway.Name].listener = listener + listeners := oasParser.GetProductionListener(gateway, resolvedListenerCerts, gwLuaScript) + gatewayLabelConfigMap[gateway.Name].listeners = listeners conf := config.ReadConfigs() if conf.Envoy.RateLimit.Enabled { gatewayLabelConfigMap[gateway.Name].customRateLimitPolicies = customRateLimitPolicies diff --git a/adapter/internal/discovery/xds/server_test.go b/adapter/internal/discovery/xds/server_test.go index 2599fc020..8b491a8c2 100644 --- a/adapter/internal/discovery/xds/server_test.go +++ b/adapter/internal/discovery/xds/server_test.go @@ -108,7 +108,7 @@ func TestUpdateAPICache(t *testing.T) { for _, label := range test.labels { SanitizeGateway(label, true) } - UpdateAPICache(test.vHosts, test.labels, test.listeners, test.adapterInternalAPI) + UpdateAPICache(test.vHosts, test.labels, test.listeners[0], "httpslistener", test.adapterInternalAPI) identifier := GetvHostsIdentifier(test.adapterInternalAPI.UUID, "prod") actualvHosts, ok := orgIDAPIvHostsMap[test.adapterInternalAPI.OrganizationID][identifier] if !ok { diff --git a/adapter/internal/oasparser/config_generator.go b/adapter/internal/oasparser/config_generator.go index acf6174ff..76ed824de 100644 --- a/adapter/internal/oasparser/config_generator.go +++ b/adapter/internal/oasparser/config_generator.go @@ -87,7 +87,7 @@ func GetGlobalClusters() ([]*clusterv3.Cluster, []*corev3.Address) { // The provided set of envoy routes will be assigned under the virtual host // // The RouteConfiguration is named as "default" -func GetProductionListener(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map[string]map[string][]byte, gwLuaScript string) *listenerv3.Listener { +func GetProductionListener(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map[string]map[string][]byte, gwLuaScript string) []*listenerv3.Listener { listeners := envoy.CreateListenerByGateway(gateway, resolvedListenerCerts, gwLuaScript) return listeners } @@ -98,10 +98,10 @@ func GetProductionListener(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map // The provided set of envoy routes will be assigned under the virtual host // // The RouteConfiguration is named as "default" -func GetRouteConfigs(vhostToRouteArrayMap map[string][]*routev3.Route, httpListener string, +func GetRouteConfigs(vhostToRouteArrayMap map[string][]*routev3.Route, routeConfigName string, customRateLimitPolicies []*model.CustomRateLimitPolicy) *routev3.RouteConfiguration { vHosts := envoy.CreateVirtualHosts(vhostToRouteArrayMap, customRateLimitPolicies) - routeConfig := envoy.CreateRoutesConfigForRds(vHosts, httpListener) + routeConfig := envoy.CreateRoutesConfigForRds(vHosts, routeConfigName) return routeConfig } @@ -110,7 +110,7 @@ func GetRouteConfigs(vhostToRouteArrayMap map[string][]*routev3.Route, httpListe // // The returned resources are listeners, clusters, routeConfigurations, endpoints func GetCacheResources(endpoints []*corev3.Address, clusters []*clusterv3.Cluster, - listeners *listenerv3.Listener, routeConfig *routev3.RouteConfiguration) ( + listeners []*listenerv3.Listener, routeConfigs []*routev3.RouteConfiguration) ( listenerRes []types.Resource, clusterRes []types.Resource, routeConfigRes []types.Resource, endpointRes []types.Resource) { @@ -122,8 +122,14 @@ func GetCacheResources(endpoints []*corev3.Address, clusters []*clusterv3.Cluste for _, endpoint := range endpoints { endpointRes = append(endpointRes, endpoint) } - listenerRes = []types.Resource{listeners} - routeConfigRes = []types.Resource{routeConfig} + listenerRes = []types.Resource{} + for _, listener := range listeners { + listenerRes = append(listenerRes, listener) + } + routeConfigRes = []types.Resource{} + for _, routeConfig := range routeConfigs { + routeConfigRes = append(routeConfigRes, routeConfig) + } return listenerRes, clusterRes, routeConfigRes, endpointRes } diff --git a/adapter/internal/oasparser/envoyconf/constants.go b/adapter/internal/oasparser/envoyconf/constants.go index f360fa8a7..d6456e410 100644 --- a/adapter/internal/oasparser/envoyconf/constants.go +++ b/adapter/internal/oasparser/envoyconf/constants.go @@ -41,10 +41,6 @@ const ( compressorFilterName string = "envoy.filters.http.compressor" ) -const ( - defaultHTTPSListenerName string = "httpslistener" -) - // cluster prefixes const ( requestInterceptClustersNamePrefix string = "reqInterceptor" diff --git a/adapter/internal/oasparser/envoyconf/listener.go b/adapter/internal/oasparser/envoyconf/listener.go index 6cc7fc5f1..43fd8d099 100644 --- a/adapter/internal/oasparser/envoyconf/listener.go +++ b/adapter/internal/oasparser/envoyconf/listener.go @@ -36,6 +36,8 @@ import ( "github.com/golang/protobuf/ptypes/wrappers" "github.com/wso2/apk/adapter/config" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" + "github.com/wso2/apk/adapter/internal/loggers" logger "github.com/wso2/apk/adapter/internal/loggers" "github.com/wso2/apk/adapter/internal/oasparser/model" "google.golang.org/protobuf/types/known/anypb" @@ -72,242 +74,263 @@ func CreateRoutesConfigForRds(vHosts []*routev3.VirtualHost, httpListeners strin // The relevant private keys and certificates (for securedListener) are fetched from the filepath // mentioned in the adapter configuration. These certificate, key values are added // as inline records (base64 encoded). -func CreateListenerByGateway(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map[string]map[string][]byte, gwLuaScript string) *listenerv3.Listener { +func CreateListenerByGateway(gateway *gwapiv1b1.Gateway, resolvedListenerCerts map[string]map[string][]byte, gwLuaScript string) []*listenerv3.Listener { conf := config.ReadConfigs() - var httpFilters []*hcmv3.HttpFilter - upgradeFilters := getUpgradeFilters() - accessLogs := getAccessLogs() - var listeners *listenerv3.Listener - var listenerName string - var listenerPort uint32 - var listenerProtocol string - var filterChains []*listenerv3.FilterChain - - for _, listenerObj := range gateway.Spec.Listeners { - if listenerObj.Name == "gatewaylistener" { - httpFilters = getHTTPFilters(gwLuaScript) - } else { - httpFilters = getHTTPFilters(` -function envoy_on_request(request_handle) -end -function envoy_on_response(response_handle) -end`) + // Prepare a map that contains all the listerners identified in all of the gateways that reconciled so far. + // This map contains port - listeners per protocol with port + protocolListenerMap := make(map[gwapiv1b1.ProtocolType]map[uint32][]gwapiv1b1.Listener) + for _, listener := range gateway.Spec.Listeners { + port := uint32(listener.Port) + if protocolListenerMap[listener.Protocol] == nil { + protocolListenerMap[listener.Protocol] = make(map[uint32][]gwapiv1b1.Listener) } - listenerPort = uint32(listenerObj.Port) - listenerProtocol = string(listenerObj.Protocol) - listenerName = defaultHTTPSListenerName - - filterChainMatch := &listenerv3.FilterChainMatch{ - ServerNames: []string{string(*listenerObj.Hostname)}, + if protocolListenerMap[listener.Protocol][port] == nil { + protocolListenerMap[listener.Protocol][port] = []gwapiv1b1.Listener{} } - - publicCertData := resolvedListenerCerts[string(listenerObj.Name)]["tls.crt"] - privateKeyData := resolvedListenerCerts[string(listenerObj.Name)]["tls.key"] - var tlsFilter *tlsv3.DownstreamTlsContext - tlsCert := generateTLSCertWithStr(string(privateKeyData), string(publicCertData)) - //TODO: Make this configurable using config map from listener object - if conf.Envoy.Downstream.TLS.MTLSAPIsEnabled { - tlsFilter = &tlsv3.DownstreamTlsContext{ - // This is false since the authentication will be done at the enforcer - RequireClientCertificate: &wrappers.BoolValue{ - Value: false, - }, - CommonTlsContext: &tlsv3.CommonTlsContext{ - //TlsCertificateSdsSecretConfigs - TlsCertificates: []*tlsv3.TlsCertificate{tlsCert}, - //For the purpose of including peer certificate into the request context - ValidationContextType: &tlsv3.CommonTlsContext_ValidationContext{ - ValidationContext: &tlsv3.CertificateValidationContext{ - TrustedCa: &corev3.DataSource{ - Specifier: &corev3.DataSource_Filename{ - Filename: conf.Envoy.Downstream.TLS.TrustedCertPath, + protocolListenerMap[listener.Protocol][port] = append(protocolListenerMap[listener.Protocol][port], listener) + } + loggers.LoggerAPKOperator.Infof("CreateListenerByGateway is called. ProtocolListenerMap: %+v", protocolListenerMap) + listenerList := make([]*listenerv3.Listener, 0) + for protocol, protocolPort := range protocolListenerMap { + for port, listeners := range protocolPort { + var httpFilters []*hcmv3.HttpFilter + upgradeFilters := getUpgradeFilters() + accessLogs := getAccessLogs() + var listener *listenerv3.Listener + var listenerName string + var filterChains []*listenerv3.FilterChain + + for _, listenerObj := range listeners { + if listenerObj.Name == "gatewaylistener" { + httpFilters = getHTTPFilters(gwLuaScript) + } else { + httpFilters = getHTTPFilters(` + function envoy_on_request(request_handle) + end + function envoy_on_response(response_handle) + end`) + } + listenerName = common.GetEnvoyListenerName(string(protocol), port) + filterChainMatch := &listenerv3.FilterChainMatch{ + ServerNames: []string{string(*listenerObj.Hostname)}, + } + var transportSocket *corev3.TransportSocket + if (protocol == gwapiv1b1.HTTPSProtocolType) { + publicCertData := resolvedListenerCerts[string(listenerObj.Name)]["tls.crt"] + privateKeyData := resolvedListenerCerts[string(listenerObj.Name)]["tls.key"] + var tlsFilter *tlsv3.DownstreamTlsContext + tlsCert := generateTLSCertWithStr(string(privateKeyData), string(publicCertData)) + //TODO: Make this configurable using config map from listener object + if conf.Envoy.Downstream.TLS.MTLSAPIsEnabled { + tlsFilter = &tlsv3.DownstreamTlsContext{ + // This is false since the authentication will be done at the enforcer + RequireClientCertificate: &wrappers.BoolValue{ + Value: false, + }, + CommonTlsContext: &tlsv3.CommonTlsContext{ + //TlsCertificateSdsSecretConfigs + TlsCertificates: []*tlsv3.TlsCertificate{tlsCert}, + //For the purpose of including peer certificate into the request context + ValidationContextType: &tlsv3.CommonTlsContext_ValidationContext{ + ValidationContext: &tlsv3.CertificateValidationContext{ + TrustedCa: &corev3.DataSource{ + Specifier: &corev3.DataSource_Filename{ + Filename: conf.Envoy.Downstream.TLS.TrustedCertPath, + }, + }, + }, + }, + }, + } + } else { + tlsFilter = &tlsv3.DownstreamTlsContext{ + CommonTlsContext: &tlsv3.CommonTlsContext{ + //TlsCertificateSdsSecretConfigs + TlsCertificates: []*tlsv3.TlsCertificate{tlsCert}, + AlpnProtocols: []string{"h2", "http/1.1"}, + }, + } + } + + marshalledTLSFilter, err := anypb.New(tlsFilter) + if err != nil { + logger.LoggerOasparser.Fatal("Error while Marshalling the downstream TLS Context for the configuration.") + } + + transportSocket = &corev3.TransportSocket{ + Name: wellknown.TransportSocketTLS, + ConfigType: &corev3.TransportSocket_TypedConfig{ + TypedConfig: marshalledTLSFilter, + }, + } + } + + var filters []*listenerv3.Filter + manager := &hcmv3.HttpConnectionManager{ + CodecType: getListenerCodecType(conf.Envoy.ListenerCodecType), + StatPrefix: httpConManagerStartPrefix, + // WebSocket upgrades enabled from the HCM + UpgradeConfigs: []*hcmv3.HttpConnectionManager_UpgradeConfig{{ + UpgradeType: "websocket", + Enabled: &wrappers.BoolValue{Value: true}, + Filters: upgradeFilters, + }}, + RouteSpecifier: &hcmv3.HttpConnectionManager_Rds{ + Rds: &hcmv3.Rds{ + RouteConfigName: common.GetEnvoyRouteConfigName(listenerName, string(listenerObj.Name)), + ConfigSource: &corev3.ConfigSource{ + ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ + Ads: &corev3.AggregatedConfigSource{}, }, + ResourceApiVersion: corev3.ApiVersion_V3, }, }, }, - }, - } - } else { - tlsFilter = &tlsv3.DownstreamTlsContext{ - CommonTlsContext: &tlsv3.CommonTlsContext{ - //TlsCertificateSdsSecretConfigs - TlsCertificates: []*tlsv3.TlsCertificate{tlsCert}, - AlpnProtocols: []string{"h2", "http/1.1"}, - }, - } - } + HttpFilters: httpFilters, + LocalReplyConfig: &hcmv3.LocalReplyConfig{ + Mappers: getErrorResponseMappers(), + }, + RequestTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.RequestTimeoutInSeconds * time.Second), // default disabled + RequestHeadersTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.RequestHeadersTimeoutInSeconds * time.Second), // default disabled + StreamIdleTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.StreamIdleTimeoutInSeconds * time.Second), // Default 5 mins + CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{ + IdleTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.IdleTimeoutInSeconds * time.Second), // Default 1 hr + }, + HttpProtocolOptions: &corev3.Http1ProtocolOptions{ + EnableTrailers: config.GetWireLogConfig().LogTrailersEnabled, + }, + UseRemoteAddress: &wrappers.BoolValue{Value: conf.Envoy.UseRemoteAddress}, + AppendXForwardedPort: true, + } - marshalledTLSFilter, err := anypb.New(tlsFilter) - if err != nil { - logger.LoggerOasparser.Fatal("Error while Marshalling the downstream TLS Context for the configuration.") - } + if len(accessLogs) > 0 { + manager.AccessLog = accessLogs + } - transportSocket := &corev3.TransportSocket{ - Name: wellknown.TransportSocketTLS, - ConfigType: &corev3.TransportSocket_TypedConfig{ - TypedConfig: marshalledTLSFilter, - }, - } + if conf.Tracing.Enabled { + if conf.Tracing.Type == TracerTypeOtlp { + if tracing, err := getTracingOTLP(conf); err == nil { + manager.Tracing = tracing + manager.GenerateRequestId = &wrappers.BoolValue{Value: conf.Tracing.Enabled} + } else { + logger.LoggerOasparser.Errorf("Failed to initialize tracing for %s. Router tracing will be disabled. Error: %s", + TracerTypeOtlp, err) + conf.Tracing.Enabled = false + } + } else if conf.Tracing.Type != TracerTypeAzure { + if tracing, err := getZipkinTracing(conf); err == nil { + manager.Tracing = tracing + manager.GenerateRequestId = &wrappers.BoolValue{Value: conf.Tracing.Enabled} + } else { + logger.LoggerOasparser.Errorf("Failed to initialize tracing for %s. Router tracing will be disabled. Error: %s", + conf.Tracing.Type, err) + conf.Tracing.Enabled = false + } + } + } - var filters []*listenerv3.Filter - manager := &hcmv3.HttpConnectionManager{ - CodecType: getListenerCodecType(conf.Envoy.ListenerCodecType), - StatPrefix: httpConManagerStartPrefix, - // WebSocket upgrades enabled from the HCM - UpgradeConfigs: []*hcmv3.HttpConnectionManager_UpgradeConfig{{ - UpgradeType: "websocket", - Enabled: &wrappers.BoolValue{Value: true}, - Filters: upgradeFilters, - }}, - RouteSpecifier: &hcmv3.HttpConnectionManager_Rds{ - Rds: &hcmv3.Rds{ - RouteConfigName: defaultHTTPSListenerName, - ConfigSource: &corev3.ConfigSource{ - ConfigSourceSpecifier: &corev3.ConfigSource_Ads{ - Ads: &corev3.AggregatedConfigSource{}, - }, - ResourceApiVersion: corev3.ApiVersion_V3, + pbst, err := anypb.New(manager) + if err != nil { + logger.LoggerOasparser.Fatal(err) + } + connectionManagerFilterP := listenerv3.Filter{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &listenerv3.Filter_TypedConfig{ + TypedConfig: pbst, }, - }, - }, - HttpFilters: httpFilters, - LocalReplyConfig: &hcmv3.LocalReplyConfig{ - Mappers: getErrorResponseMappers(), - }, - RequestTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.RequestTimeoutInSeconds * time.Second), // default disabled - RequestHeadersTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.RequestHeadersTimeoutInSeconds * time.Second), // default disabled - StreamIdleTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.StreamIdleTimeoutInSeconds * time.Second), // Default 5 mins - CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{ - IdleTimeout: ptypes.DurationProto(conf.Envoy.Connection.Timeouts.IdleTimeoutInSeconds * time.Second), // Default 1 hr - }, - HttpProtocolOptions: &corev3.Http1ProtocolOptions{ - EnableTrailers: config.GetWireLogConfig().LogTrailersEnabled, - }, - UseRemoteAddress: &wrappers.BoolValue{Value: conf.Envoy.UseRemoteAddress}, - AppendXForwardedPort: true, - } - - if len(accessLogs) > 0 { - manager.AccessLog = accessLogs - } - - if conf.Tracing.Enabled { - if conf.Tracing.Type == TracerTypeOtlp { - if tracing, err := getTracingOTLP(conf); err == nil { - manager.Tracing = tracing - manager.GenerateRequestId = &wrappers.BoolValue{Value: conf.Tracing.Enabled} - } else { - logger.LoggerOasparser.Errorf("Failed to initialize tracing for %s. Router tracing will be disabled. Error: %s", - TracerTypeOtlp, err) - conf.Tracing.Enabled = false } - } else if conf.Tracing.Type != TracerTypeAzure { - if tracing, err := getZipkinTracing(conf); err == nil { - manager.Tracing = tracing - manager.GenerateRequestId = &wrappers.BoolValue{Value: conf.Tracing.Enabled} + + // add filters + filters = append(filters, &connectionManagerFilterP) + if protocol == gwapiv1b1.HTTPSProtocolType { + filterChains = append(filterChains, &listenerv3.FilterChain{ + FilterChainMatch: filterChainMatch, + Filters: filters, + TransportSocket: transportSocket, + }) } else { - logger.LoggerOasparser.Errorf("Failed to initialize tracing for %s. Router tracing will be disabled. Error: %s", - conf.Tracing.Type, err) - conf.Tracing.Enabled = false + filterChains = append(filterChains, &listenerv3.FilterChain{ + Filters: filters, + }) } - } - } - pbst, err := anypb.New(manager) - if err != nil { - logger.LoggerOasparser.Fatal(err) - } - connectionManagerFilterP := listenerv3.Filter{ - Name: wellknown.HTTPConnectionManager, - ConfigType: &listenerv3.Filter_TypedConfig{ - TypedConfig: pbst, - }, - } - - // add filters - filters = append(filters, &connectionManagerFilterP) - filterChains = append(filterChains, &listenerv3.FilterChain{ - FilterChainMatch: filterChainMatch, - Filters: filters, - TransportSocket: transportSocket, - }) - - } + } - if gwapiv1b1.ProtocolType(listenerProtocol) == gwapiv1b1.HTTPSProtocolType { - listenerHostAddress := defaultListenerHostAddress - securedListenerAddress := &corev3.Address_SocketAddress{ - SocketAddress: &corev3.SocketAddress{ - Protocol: corev3.SocketAddress_TCP, - Address: listenerHostAddress, - PortSpecifier: &corev3.SocketAddress_PortValue{ - PortValue: uint32(listenerPort), - }, - }, - } + if protocol == gwapiv1b1.HTTPSProtocolType { + listenerHostAddress := defaultListenerHostAddress + securedListenerAddress := &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Protocol: corev3.SocketAddress_TCP, + Address: listenerHostAddress, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + } - //var tlsInspector *tlsInspectorv3.TlsInspector + //var tlsInspector *tlsInspectorv3.TlsInspector - tlsInspector := &tlsInspectorv3.TlsInspector{} - marshalledListenerFilter, err := anypb.New(tlsInspector) - if err != nil { - logger.LoggerOasparser.Fatal("Error while Marshalling the TlsInspector for the configuration.") - } + tlsInspector := &tlsInspectorv3.TlsInspector{} + marshalledListenerFilter, err := anypb.New(tlsInspector) + if err != nil { + logger.LoggerOasparser.Fatal("Error while Marshalling the TlsInspector for the configuration.") + } - listenerFilters := []*listenerv3.ListenerFilter{ - { // TLS Inspector - Name: wellknown.TlsInspector, - ConfigType: &listenerv3.ListenerFilter_TypedConfig{ - TypedConfig: marshalledListenerFilter, - }, - }, - } + listenerFilters := []*listenerv3.ListenerFilter{ + { // TLS Inspector + Name: wellknown.TlsInspector, + ConfigType: &listenerv3.ListenerFilter_TypedConfig{ + TypedConfig: marshalledListenerFilter, + }, + }, + } - securedListener := listenerv3.Listener{ - Name: string(listenerName), - Address: &corev3.Address{ - Address: securedListenerAddress, - }, - ListenerFilters: listenerFilters, - FilterChains: filterChains, - } + listener = &listenerv3.Listener{ + Name: string(listenerName), + Address: &corev3.Address{ + Address: securedListenerAddress, + }, + ListenerFilters: listenerFilters, + FilterChains: filterChains, + } + logger.LoggerOasparser.Infof("Secured Listener is added. %s : %d", listenerHostAddress, port) + } else { + logger.LoggerOasparser.Info("No SecuredListenerPort is included.") + } - listeners = &securedListener - logger.LoggerOasparser.Infof("Secured Listener is added. %s : %d", listenerHostAddress, uint32(listenerPort)) - } else { - logger.LoggerOasparser.Info("No SecuredListenerPort is included.") - } + if protocol == gwapiv1b1.HTTPProtocolType { + listenerHostAddress := defaultListenerHostAddress + listenerAddress := &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Protocol: corev3.SocketAddress_TCP, + Address: listenerHostAddress, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + } - if gwapiv1b1.ProtocolType(listenerProtocol) == gwapiv1b1.HTTPProtocolType { - listenerHostAddress := defaultListenerHostAddress - listenerAddress := &corev3.Address_SocketAddress{ - SocketAddress: &corev3.SocketAddress{ - Protocol: corev3.SocketAddress_TCP, - Address: listenerHostAddress, - PortSpecifier: &corev3.SocketAddress_PortValue{ - PortValue: uint32(listenerPort), - }, - }, - } + listener = &listenerv3.Listener{ + Name: string(listenerName), + Address: &corev3.Address{ + Address: listenerAddress, + }, + FilterChains: filterChains, + } + logger.LoggerOasparser.Infof("Non-secured Listener is added. %s : %d", listenerHostAddress, port) + } else { + logger.LoggerOasparser.Info("No Non-securedListenerPort is included.") + } - listener := listenerv3.Listener{ - Name: string(listenerName), - Address: &corev3.Address{ - Address: listenerAddress, - }, - FilterChains: filterChains, + if listeners == nil { + err := errors.New("No Listeners are configured as no port value is mentioned under securedListenerPort or ListenerPort") + logger.LoggerOasparser.Fatal(err) + } + listenerList = append(listenerList, listener) } - listeners = &listener - logger.LoggerOasparser.Infof("Non-secured Listener is added. %s : %d", listenerHostAddress, uint32(listenerPort)) - } else { - logger.LoggerOasparser.Info("No Non-securedListenerPort is included.") - } - - if listeners == nil { - err := errors.New("No Listeners are configured as no port value is mentioned under securedListenerPort or ListenerPort") - logger.LoggerOasparser.Fatal(err) } - return listeners + logger.LoggerOasparser.Infof("Listener list size. %+v", len(listenerList)) + return listenerList } // CreateVirtualHosts creates VirtualHost configurations for envoy which serves diff --git a/adapter/internal/oasparser/envoyconf/listener_test.go b/adapter/internal/oasparser/envoyconf/listener_test.go index e889f668c..bcd5ae03d 100644 --- a/adapter/internal/oasparser/envoyconf/listener_test.go +++ b/adapter/internal/oasparser/envoyconf/listener_test.go @@ -46,7 +46,7 @@ func TestCreateListenerWithRds(t *testing.T) { listeners := CreateListenerByGateway(gateway, resolvedListenerCerts, "") assert.NotEmpty(t, listeners, "Listeners creation has been failed") - securedListener := listeners + securedListener := listeners[0] if securedListener.Validate() != nil { t.Error("Listener validation failed") } diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go index 0b9f67ba5..936be2f66 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/wso2/apk/adapter/internal/data_holder" envoy "github.com/wso2/apk/adapter/internal/oasparser/envoyconf" "github.com/wso2/apk/adapter/internal/operator/constants" "github.com/wso2/apk/adapter/internal/operator/synchronizer" @@ -99,6 +100,24 @@ func TestCreateRoutesWithClustersWithExactAndRegularExpressionRules(t *testing.T }, }, } + hostName := gwapiv1b1.Hostname("prod.gw.wso2.com") + gateway := gwapiv1b1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "default-gateway", + }, + Spec: gwapiv1b1.GatewaySpec{ + Listeners: []gwapiv1b1.Listener{ + { + Name: "httpslistener", + Hostname: &hostName, + Protocol: gwapiv1b1.HTTPSProtocolType, + }, + }, + }, + } + + data_holder.UpdateGateway(gateway) httpRouteState.HTTPRouteCombined = &httpRoute diff --git a/adapter/internal/operator/controllers/dp/gateway_controller.go b/adapter/internal/operator/controllers/dp/gateway_controller.go index eb2da93ea..cdcdfc705 100644 --- a/adapter/internal/operator/controllers/dp/gateway_controller.go +++ b/adapter/internal/operator/controllers/dp/gateway_controller.go @@ -224,6 +224,9 @@ func (gatewayReconciler *GatewayReconciler) resolveGatewayState(ctx context.Cont namespace := gwapiv1b1.Namespace(gateway.Namespace) // Retireve listener Certificates for _, listener := range gateway.Spec.Listeners { + if listener.Protocol == gwapiv1b1.HTTPProtocolType { + continue + } data, err := gatewayReconciler.resolveListenerSecretRefs(ctx, &listener.TLS.CertificateRefs[0], string(namespace)) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3105, logging.BLOCKER, "Error resolving listener certificates: %v", err)) diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index a0d32ff14..4630190b4 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -83,7 +83,7 @@ func InitOperator() { flag.Parse() log.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - operatorDataStore := synchronizer.CreateNewOperatorDataStore() + operatorDataStore := synchronizer.GetOperatorDataStore() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 4660c3159..97c2e080d 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -34,14 +34,20 @@ type OperatorDataStore struct { mu sync.Mutex } -// CreateNewOperatorDataStore creates a new OperatorDataStore. -func CreateNewOperatorDataStore() *OperatorDataStore { - return &OperatorDataStore{ +var operatorDataStore *OperatorDataStore + +func init() { + operatorDataStore = &OperatorDataStore{ apiStore: map[types.NamespacedName]*APIState{}, gatewayStore: map[types.NamespacedName]*GatewayState{}, } } +// GetOperatorDataStore creates a new OperatorDataStore. +func GetOperatorDataStore() *OperatorDataStore { + return operatorDataStore +} + // AddAPIState stores a new API in the OperatorDataStore. func (ods *OperatorDataStore) AddAPIState(apiNamespacedName types.NamespacedName, apiState *APIState) { ods.mu.Lock() diff --git a/adapter/internal/operator/synchronizer/gateway_synchronizer.go b/adapter/internal/operator/synchronizer/gateway_synchronizer.go index 200a8fa07..81c7bb3af 100644 --- a/adapter/internal/operator/synchronizer/gateway_synchronizer.go +++ b/adapter/internal/operator/synchronizer/gateway_synchronizer.go @@ -21,6 +21,7 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" "github.com/wso2/apk/adapter/config" + dataHolder "github.com/wso2/apk/adapter/internal/data_holder" "github.com/wso2/apk/adapter/internal/discovery/xds" "github.com/wso2/apk/adapter/internal/interceptor" "github.com/wso2/apk/adapter/internal/loggers" @@ -81,6 +82,7 @@ func undeployGateway(gatewayState GatewayState) error { var err error if gatewayState.GatewayDefinition != nil { _, err = DeleteGateway(gatewayState.GatewayDefinition) + dataHolder.RemoveGateway(*gatewayState.GatewayDefinition) } return err } @@ -88,6 +90,7 @@ func undeployGateway(gatewayState GatewayState) error { // AddOrUpdateGateway adds/update a Gateway to the XDS server. func AddOrUpdateGateway(gatewayState GatewayState, state string) (string, error) { gateway := gatewayState.GatewayDefinition + dataHolder.UpdateGateway(*gateway) xds.SanitizeGateway(gateway.Name, true) resolvedListenerCerts := gatewayState.GatewayStateData.GatewayResolvedListenerCerts customRateLimitPolicies := getCustomRateLimitPolicies(gatewayState.GatewayStateData.GatewayCustomRateLimitPolicies) diff --git a/adapter/internal/operator/synchronizer/synchronizer.go b/adapter/internal/operator/synchronizer/synchronizer.go index 3dc7e8e5d..b23114941 100644 --- a/adapter/internal/operator/synchronizer/synchronizer.go +++ b/adapter/internal/operator/synchronizer/synchronizer.go @@ -21,11 +21,14 @@ import ( "bytes" "crypto/tls" "encoding/json" + "errors" "fmt" "net/http" "time" + "github.com/wso2/apk/adapter/internal/data_holder" "github.com/wso2/apk/adapter/internal/discovery/xds" + "github.com/wso2/apk/adapter/internal/discovery/xds/common" "k8s.io/apimachinery/pkg/types" "github.com/wso2/apk/adapter/config" @@ -189,13 +192,23 @@ func GenerateAdapterInternalAPI(apiState APIState, httpRoute *HTTPRouteState, en } vHosts := getVhostsForAPI(httpRoute.HTTPRouteCombined) labels := getLabelsForAPI(httpRoute.HTTPRouteCombined) - listeners := getListenersForAPI(httpRoute.HTTPRouteCombined, adapterInternalAPI.UUID) - - err := xds.UpdateAPICache(vHosts, labels, listeners, adapterInternalAPI) - if err != nil { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MAJOR, "Error updating the API : %s:%s in vhosts: %s, API_UUID: %v. %v", - adapterInternalAPI.GetTitle(), adapterInternalAPI.GetVersion(), vHosts, adapterInternalAPI.UUID, err)) + listeners, relativeSectionNames := getListenersForAPI(httpRoute.HTTPRouteCombined, adapterInternalAPI.UUID) + // We dont have a use case where a perticular API's two different http routes refer to two different gateway. Hence get the first listener name for the list for processing. + if len(listeners) == 0 || len(relativeSectionNames) == 0 { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MINOR, "Failed to find a matching listener for http route: %v. ", + httpRoute.HTTPRouteCombined.Name)) + return nil, errors.New("failed to find matching listener name for the provided http route") + } + listenerName := listeners[0] + sectionName := relativeSectionNames[0] + if len(listeners) != 0 { + err := xds.UpdateAPICache(vHosts, labels, listenerName, sectionName, adapterInternalAPI) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2633, logging.MAJOR, "Error updating the API : %s:%s in vhosts: %s, API_UUID: %v. %v", + adapterInternalAPI.GetTitle(), adapterInternalAPI.GetVersion(), vHosts, adapterInternalAPI.UUID, err)) + } } + return &adapterInternalAPI, nil } @@ -225,12 +238,35 @@ func getLabelsForAPI(httpRoute *gwapiv1b1.HTTPRoute) []string { } // getListenersForAPI returns the listeners related to an API. -func getListenersForAPI(httpRoute *gwapiv1b1.HTTPRoute, apiUUID string) []string { +func getListenersForAPI(httpRoute *gwapiv1b1.HTTPRoute, apiUUID string) ([]string, []string) { var listeners []string + var sectionNames []string for _, parentRef := range httpRoute.Spec.ParentRefs { - listeners = append(listeners, string(*parentRef.SectionName)) + namespace := httpRoute.GetNamespace() + if parentRef.Namespace != nil && *parentRef.Namespace != "" { + namespace = string(*parentRef.Namespace) + } + gateway, found := data_holder.GetGatewayMap()[types.NamespacedName{ + Namespace: namespace, + Name: string(parentRef.Name), + }.String()] + if found { + // find the matching listener + matchedListener, listenerFound := common.FindElement(gateway.Spec.Listeners, func(listener gwapiv1b1.Listener) bool { + if string(listener.Name) == string(*parentRef.SectionName) { + return true + } + return false + }) + if listenerFound { + sectionNames = append(sectionNames, string(matchedListener.Name)) + listeners = append(listeners, common.GetEnvoyListenerName(string(matchedListener.Protocol), uint32(matchedListener.Port))) + continue + } + } + loggers.LoggerAPKOperator.Errorf("Failed to find matching listeners for the httproute: %+v", httpRoute.Name) } - return listeners + return listeners, sectionNames } // Runtime client connetion diff --git a/helm-charts/README.md b/helm-charts/README.md index bc7bb14ba..31bcb0c0a 100644 --- a/helm-charts/README.md +++ b/helm-charts/README.md @@ -41,6 +41,9 @@ A Helm chart for APK components | wso2.apk.idp.signing.fileName | string | `""` | IDP jwt signing certificate file name | | wso2.apk.dp.enabled | bool | `true` | Enable the deployment of the Data Plane | | wso2.apk.dp.environment.name | string | `Default` | Environment of the Data Plane | +| wso2.apk.dp.gateway.httpListener.enabled | bool | `false` | HTTP listener enabled or not | +| wso2.apk.dp.gateway.httpListener.hostname | string | `"api.am.wso2.com"` | HTTP listener hostname | +| wso2.apk.dp.gateway.httpListener.port | int | `9080` | HTTP listener port | | wso2.apk.dp.gateway.listener.hostname | string | `"gw.wso2.com"` | Gateway Listener Hostname | | wso2.apk.dp.gateway.listener.secretName | string | `""` | Gateway Listener Certificate Secret Name | | wso2.apk.dp.gateway.listener.dns | list | `["*.gw.wso2.com","*.sandbox.gw.wso2.com","prod.gw.wso2.com"]` | DNS entries for gateway listener certificate | diff --git a/helm-charts/templates/data-plane/config-deployer/config-api-route.yaml b/helm-charts/templates/data-plane/config-deployer/config-api-route.yaml index ac84d2aaf..d20be1192 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-api-route.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-api-route.yaml @@ -48,4 +48,4 @@ spec: kind: "Gateway" name: "default" sectionName: "httpslistener" -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm-charts/templates/data-plane/config-deployer/config-deploy-api-route.yaml b/helm-charts/templates/data-plane/config-deployer/config-deploy-api-route.yaml index 2a242bb00..a33d1a162 100644 --- a/helm-charts/templates/data-plane/config-deployer/config-deploy-api-route.yaml +++ b/helm-charts/templates/data-plane/config-deployer/config-deploy-api-route.yaml @@ -60,4 +60,4 @@ spec: kind: "Gateway" name: "default" sectionName: "httpslistener" -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm-charts/templates/data-plane/gateway-components/adapter/gateway.yaml b/helm-charts/templates/data-plane/gateway-components/adapter/gateway.yaml index 5ba32b819..e1f8b68b3 100644 --- a/helm-charts/templates/data-plane/gateway-components/adapter/gateway.yaml +++ b/helm-charts/templates/data-plane/gateway-components/adapter/gateway.yaml @@ -9,8 +9,14 @@ metadata: spec: gatewayClassName: "default" listeners: + {{ if and .Values.wso2.apk.dp.gateway.httpListener .Values.wso2.apk.dp.gateway.httpListener.enabled }} + - name: httplistener + hostname: "{{ .Values.wso2.apk.dp.gateway.httpListener.hostname | default "api.am.wso2.com"}}" + port: {{ .Values.wso2.apk.dp.gateway.httpListener.port | default 9080}} + protocol: HTTP + {{ end }} {{ if or .Values.wso2.apk.dp.enabled .Values.wso2.apk.cp.enabled }} - - name: apilistener + - name: httpslistener hostname: "{{ .Values.wso2.apk.listener.hostname | default "api.am.wso2.com"}}" port: 9095 protocol: "HTTPS" diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 98d3dc5f0..33bf221d9 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -238,6 +238,10 @@ spec: image: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.router.image }} imagePullPolicy: {{ .Values.wso2.apk.dp.gatewayRuntime.deployment.router.imagePullPolicy }} ports: + {{ if and .Values.wso2.apk.dp.gateway.httpListener .Values.wso2.apk.dp.gateway.httpListener.enabled }} + - containerPort: {{ .Values.wso2.apk.dp.gateway.httpListener.port | default 9080}} + protocol: "TCP" + {{ end }} - containerPort: 9095 protocol: "TCP" - containerPort: 9090 diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml index dbf27dd56..dca2ff7ed 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-service.yaml @@ -30,7 +30,12 @@ spec: selector: {{ include "apk-helm.pod.selectorLabels" (dict "root" . "app" "gateway" ) | indent 4}} ports: - - name: endpoint1 - protocol: TCP - port: 9095 +{{ if and .Values.wso2.apk.dp.gateway.httpListener .Values.wso2.apk.dp.gateway.httpListener.enabled }} + - name: "http-endpoint" + protocol: TCP + port: {{ .Values.wso2.apk.dp.gateway.httpListener.port | default 9080 }} +{{ end }} + - name: "https-endpoint" + protocol: TCP + port: 9095 {{- end -}} diff --git a/helm-charts/templates/idp/authenticationEndpoint-domain-api-httproute.yaml b/helm-charts/templates/idp/authenticationEndpoint-domain-api-httproute.yaml index 1c9aefd13..ab634ea44 100644 --- a/helm-charts/templates/idp/authenticationEndpoint-domain-api-httproute.yaml +++ b/helm-charts/templates/idp/authenticationEndpoint-domain-api-httproute.yaml @@ -38,5 +38,5 @@ spec: - group: "gateway.networking.k8s.io" kind: "Gateway" name: "default" - sectionName: "httpslistener" + sectionName: "idplistener" {{- end -}} diff --git a/helm-charts/templates/idp/commonoauth-domain-api-httproute.yaml b/helm-charts/templates/idp/commonoauth-domain-api-httproute.yaml index e52b71f33..471312370 100644 --- a/helm-charts/templates/idp/commonoauth-domain-api-httproute.yaml +++ b/helm-charts/templates/idp/commonoauth-domain-api-httproute.yaml @@ -38,5 +38,5 @@ spec: - group: "gateway.networking.k8s.io" kind: "Gateway" name: "default" - sectionName: "httpslistener" + sectionName: "idplistener" {{- end -}} diff --git a/helm-charts/templates/idp/dcr-domain-api-httproute.yaml b/helm-charts/templates/idp/dcr-domain-api-httproute.yaml index d12cbc671..ce2b00e4d 100644 --- a/helm-charts/templates/idp/dcr-domain-api-httproute.yaml +++ b/helm-charts/templates/idp/dcr-domain-api-httproute.yaml @@ -38,5 +38,5 @@ spec: - group: "gateway.networking.k8s.io" kind: "Gateway" name: "default" - sectionName: "httpslistener" + sectionName: "idplistener" {{- end -}} diff --git a/helm-charts/templates/idp/oauth-domain-api-httproute.yaml b/helm-charts/templates/idp/oauth-domain-api-httproute.yaml index e406e83fb..5bfd35ee9 100644 --- a/helm-charts/templates/idp/oauth-domain-api-httproute.yaml +++ b/helm-charts/templates/idp/oauth-domain-api-httproute.yaml @@ -38,5 +38,5 @@ spec: - group: "gateway.networking.k8s.io" kind: "Gateway" name: "default" - sectionName: "httpslistener" + sectionName: "idplistener" {{- end -}} diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template index 2b2ccaca3..0a80f6ee3 100644 --- a/helm-charts/values.yaml.template +++ b/helm-charts/values.yaml.template @@ -85,6 +85,13 @@ wso2: - "*.gw.wso2.com" - "*.sandbox.gw.wso2.com" - "prod.gw.wso2.com" + httpListener: + # -- HTTP listener enabled or not + enabled: false + # -- HTTP listener hostname + hostname: "api.am.wso2.com" + # -- HTTP listener port + port: 9080 autoscaling: # -- Enable autoscaling for Gateway enabled: false diff --git a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/APKGenerationSteps.java b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/APKGenerationSteps.java index 42e91f8e4..21f2b09a0 100644 --- a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/APKGenerationSteps.java +++ b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/APKGenerationSteps.java @@ -62,7 +62,7 @@ public void generate_the_apk_conf_file(String apiType) throws Exception { .addPart("definition", new FileBody(definitionFile)); HttpEntity multipartEntity = builder.build(); - HttpResponse httpResponse = sharedContext.getHttpClient().doPostWithMultipart(Utils.getConfigGeneratorURL(), + HttpResponse httpResponse = sharedContext.getHttpClient().doPostWithMultipart(Utils.getConfigGeneratorURL(), multipartEntity); sharedContext.setResponse(httpResponse); } @@ -73,6 +73,6 @@ public void the_response_body_should_be_in_resources(String expectedAPKConfFileP URL url = Resources.getResource(expectedAPKConfFilePath); String text = Resources.toString(url, StandardCharsets.UTF_8); - Assert.assertEquals(text, sharedContext.getHttpClient().getResponsePayload(sharedContext.getResponse())); + Assert.assertEquals(sharedContext.getHttpClient().getResponsePayload(sharedContext.getResponse()), text); } } diff --git a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature index cf70177ce..b96dff960 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature @@ -51,7 +51,7 @@ Feature: API Definition Endpoint Then the response status code should be 404 And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/api-definition" with body "" Then the response status code should be 404 - + Scenario: Testing a deleted production endpoint Given The system is ready And I have a valid subscription diff --git a/test/integration/integration/tests/deploy-api-to-multiple-ports-listener.go b/test/integration/integration/tests/deploy-api-to-multiple-ports-listener.go new file mode 100644 index 000000000..5aca87f71 --- /dev/null +++ b/test/integration/integration/tests/deploy-api-to-multiple-ports-listener.go @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tests + +import ( + "testing" + + "github.com/wso2/apk/test/integration/integration/utils/http" + "github.com/wso2/apk/test/integration/integration/utils/suite" +) + +func init() { + IntegrationTests = append(IntegrationTests, APIDifferentPortListener) +} + +// APIDifferentPortListener test +var APIDifferentPortListener = suite.IntegrationTest{ + ShortName: "APIDifferentPortListener", + Description: "An API is deployed to a different listener other than default gateway listener", + Manifests: []string{"tests/multiple_port_listener.yaml"}, + Test: func(t *testing.T, suite *suite.IntegrationTestSuite) { + ns := "gateway-integration-test-infra" + gwAddr := "api.am.wso2.com:9080" + token := http.GetTestToken(t) + + testCases := []http.ExpectedResponse{ + { + Request: http.Request{ + Host: "api.am.wso2.com", + Path: "/test-api-with-multiple-port-listener/v1.0.0/user/user123/playlist/watch-later", + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/user/user123/playlist/watch-later", + }, + }, + Backend: "infra-backend-v1", + Namespace: ns, + }, + { + Request: http.Request{ + Host: "api.am.wso2.com", + Path: "/test-api-with-multiple-port-listener-1/v1.0.0/user/user123/playlist/watch-later", + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/user/user123/playlist/watch-later", + }, + }, + Backend: "infra-backend-v1", + Namespace: ns, + }, + } + for i := range testCases { + tc := testCases[i] + tc.Request.Headers = http.AddBearerTokenToHeader(token, tc.Request.Headers) + t.Run(tc.GetTestCaseName(i), func(t *testing.T) { + t.Parallel() + http.MakeHTTPRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc) + }) + } + }, +} diff --git a/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml b/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml new file mode 100644 index 000000000..72ada918d --- /dev/null +++ b/test/integration/integration/tests/resources/tests/multiple_port_listener.yaml @@ -0,0 +1,88 @@ +# Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +# +# WSO2 LLC. licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: dp.wso2.com/v1alpha1 +kind: API +metadata: + name: test-multiple-port-listener + namespace: gateway-integration-test-infra +spec: + apiName: API with Different Listener + apiType: REST + apiVersion: 1.0.0 + basePath: /test-api-with-multiple-port-listener/v1.0.0 + #definitionFileRef: definition-file + production: + - httpRouteRefs: + - test-api-with-multiple-port-listener-httproute + organization: wso2-org +--- +apiVersion: dp.wso2.com/v1alpha1 +kind: API +metadata: + name: test-multiple-port-listener-1 + namespace: gateway-integration-test-infra +spec: + apiName: API with Different Listener + apiType: REST + apiVersion: 1.0.0 + basePath: /test-api-with-multiple-port-listener-1/v1.0.0 + #definitionFileRef: definition-file + production: + - httpRouteRefs: + - test-api-with-multiple-port-listener-httproute + organization: wso2-org +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: test-api-with-multiple-port-listener-httproute + namespace: gateway-integration-test-infra +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: default + namespace: apk-integration-test + sectionName: httplistener + hostnames: + - api.am.wso2.com + rules: + - matches: + - path: + type: RegularExpression + value: /user/([^/]+)/playlist/([^/]+) + method: GET + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplaceFullPath + replaceFullPath: /user/\1/playlist/\2 + backendRefs: + - group: dp.wso2.com + kind: Backend + name: infra-backend-v1 +--- +apiVersion: dp.wso2.com/v1alpha1 +kind: Backend +metadata: + name: infra-backend-v1 + namespace: gateway-integration-test-infra +spec: + services: + - host: infra-backend-v1.gateway-integration-test-infra + port: 8080 diff --git a/test/integration/integration/utils/http/http.go b/test/integration/integration/utils/http/http.go index 7d35055a5..6e4d387f5 100644 --- a/test/integration/integration/utils/http/http.go +++ b/test/integration/integration/utils/http/http.go @@ -109,6 +109,19 @@ func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripp WaitForConsistentResponse(t, r, req, expected, timeoutConfig.RequiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency) } +// MakeHTTPRequestAndExpectEventuallyConsistentResponse makes a http request with the given parameters, +// understanding that the request may fail for some amount of time. +// +// Once the request succeeds consistently with the response having the expected status code, make +// additional assertions on the response body using the provided ExpectedResponse. +func MakeHTTPRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, expected ExpectedResponse) { + t.Helper() + + req := MakeRequest(t, &expected, gwAddr, "HTTP", "http") + + WaitForConsistentResponse(t, r, req, expected, timeoutConfig.RequiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency) +} + // MakeRequest make a request with the given parameters. func MakeRequest(t *testing.T, expected *ExpectedResponse, gwAddr, protocol, scheme string) roundtripper.Request { t.Helper()