From ef7bc04c5ce28c370517c3ddde96f52c179cfa0e Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Tue, 27 Aug 2024 09:38:03 +0530 Subject: [PATCH 01/24] Enforcer initial changes --- adapter/go.mod | 32 +- adapter/go.sum | 82 ++-- .../internal/oasparser/envoyconf/constants.go | 1 + .../oasparser/envoyconf/http_filters.go | 44 +++ .../oasparser/envoyconf/routes_configs.go | 153 +++++++- .../envoyconf/routes_with_clusters.go | 116 +++++- .../oasparser/model/adapter_internal_api.go | 29 +- adapter/internal/oasparser/model/resource.go | 36 +- adapter/internal/operator/PROJECT | 9 + .../operator/config/crd/kustomization.yaml | 1 + ...cainjection_in_dp_airatelimitpolicies.yaml | 7 + .../webhook_in_dp_airatelimitpolicies.yaml | 16 + .../dp_airatelimitpolicy_editor_role.yaml | 31 ++ .../dp_airatelimitpolicy_viewer_role.yaml | 27 ++ .../internal/operator/constants/constants.go | 9 +- .../operator/controllers/dp/api_controller.go | 235 +++++++++++- .../operator/controllers/dp/suite_test.go | 4 + adapter/internal/operator/operator.go | 3 + .../operator/synchronizer/api_state.go | 2 + .../operator/synchronizer/data_store.go | 42 ++ .../operator/synchronizer/rest_api.go | 8 +- adapter/internal/operator/utils/utils.go | 5 + common-controller/go.mod | 36 +- common-controller/go.sum | 90 ++--- common-controller/internal/cache/datastore.go | 26 ++ .../operator/config/crd/kustomization.yaml | 1 + .../dp/airatelimitpolicy_controller.go | 159 ++++++++ .../operator/controllers/dp/suite_test.go | 8 + .../internal/operator/operator.go | 5 + .../internal/xds/ratelimiter_cache.go | 84 +++- common-controller/internal/xds/server.go | 11 + common-go-libs/PROJECT | 13 + .../dp/v1alpha3/airatelimitpolicy_types.go | 98 +++++ .../apis/dp/v1alpha3/zz_generated.deepcopy.go | 140 +++++++ .../dp.wso2.com_airatelimitpolicies.yaml | 185 +++++++++ common-go-libs/config/crd/kustomization.yaml | 3 + ...cainjection_in_dp_airatelimitpolicies.yaml | 7 + .../webhook_in_dp_airatelimitpolicies.yaml | 16 + .../dp_airatelimitpolicy_editor_role.yaml | 31 ++ .../dp_airatelimitpolicy_viewer_role.yaml | 27 ++ common-go-libs/config/rbac/role.yaml | 32 ++ .../dp_v1alpha3_airatelimitpolicy.yaml | 12 + common-go-libs/constants/constant.go | 1 + .../apk/enforcer/config/ConfigHolder.java | 2 + .../apk/enforcer/config/EnvVarConfig.java | 21 + .../grpc/ExternalProcessorService.java | 303 +++++++++++++++ .../enforcer/grpc/client/RatelimitClient.java | 97 +++++ .../wso2/apk/enforcer/server/AuthServer.java | 3 + gateway/router/Dockerfile | 2 +- .../crds/dp.wso2.com_airatelimitpolicies.yaml | 201 ++++++++++ .../gateway-runtime-deployment.yaml | 24 +- .../serviceAccount/apk-cluster-role.yaml | 9 + libs.versions.toml | 4 +- .../tests/api/APIDefinitionEndpoint.feature | 84 ---- .../tests/api/APISubscription.feature | 72 ---- .../resources/tests/api/BackendRetry.feature | 35 -- .../tests/api/BackendTimeout.feature | 48 --- .../resources/tests/api/BasicAuth.feature | 43 --- .../BasicDeploymentAndApiInvocation.feature | 138 ------- .../src/test/resources/tests/api/CORS.feature | 60 --- .../tests/api/CircuitBreaker.feature | 36 -- .../api/CircuitBreakerEnvoyConfigDump.feature | 26 -- .../api/CircuitBreakerMaxRequest.feature | 47 --- .../tests/api/CustomRatelimit.feature | 118 ------ .../DifferentEndpointResourceLevel.feature | 28 -- .../api/DifferentSandProdEndpoint.feature | 34 -- .../test/resources/tests/api/Endpoint.feature | 37 -- .../tests/api/GlobalInterceptor.feature | 25 -- .../test/resources/tests/api/GraphQL.feature | 237 ------------ .../tests/api/HeaderModifier.feature | 84 ---- .../resources/tests/api/Interceptor.feature | 119 ------ .../src/test/resources/tests/api/JWT.feature | 126 ------ .../tests/api/MTLSwithOAuth2Mandatory.feature | 358 ------------------ .../tests/api/MTLSwithOAuth2Optional.feature | 180 --------- .../tests/api/MultiEnvironment.feature | 117 ------ .../tests/api/OrganizationBasedAPIS.feature | 47 --- .../resources/tests/api/RequestMirror.feature | 21 - .../tests/api/RequestRedirect.feature | 51 --- .../resources/tests/api/RevokedToken.feature | 35 -- .../tests/api/SemanticVersioning.feature | 118 ------ .../tests/api/SimpleRateLimit.feature | 76 ---- .../resources/tests/api/deployment.feature | 44 --- .../tests/api/resourceInterceptor.feature | 34 -- 83 files changed, 2340 insertions(+), 2681 deletions(-) create mode 100644 adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml create mode 100644 adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml create mode 100644 adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml create mode 100644 adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml create mode 100644 common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go create mode 100644 common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go create mode 100644 common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml create mode 100644 common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml create mode 100644 common-go-libs/config/rbac/role.yaml create mode 100644 common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml create mode 100644 gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java create mode 100644 helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/CORS.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/JWT.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/deployment.feature delete mode 100644 test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature diff --git a/adapter/go.mod b/adapter/go.mod index eeb7f4350b..bf720480fd 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -3,9 +3,9 @@ module github.com/wso2/apk/adapter go 1.22 require ( - github.com/envoyproxy/go-control-plane v0.12.0 + github.com/envoyproxy/go-control-plane v0.13.0 github.com/fsnotify/fsnotify v1.7.0 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 @@ -14,7 +14,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb - google.golang.org/grpc v1.62.0 + google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.29.2 @@ -24,11 +24,12 @@ require ( ) require ( + cel.dev/expr v0.15.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -56,9 +57,10 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect @@ -70,17 +72,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.2 // indirect @@ -99,8 +99,8 @@ replace github.com/wso2/apk/common-go-libs => ../common-go-libs require ( github.com/ghodss/yaml v1.0.0 - github.com/stretchr/testify v1.8.4 - golang.org/x/sys v0.17.0 // indirect + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.20.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 sigs.k8s.io/gateway-api v1.0.0 ) diff --git a/adapter/go.sum b/adapter/go.sum index 9bc0af125c..5eaa7328cd 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= @@ -9,13 +11,13 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -25,8 +27,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= -github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= @@ -57,13 +59,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -112,6 +111,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -119,8 +120,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -149,8 +150,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -159,7 +161,6 @@ github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqf github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -171,51 +172,39 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,7 +212,6 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -232,18 +220,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/adapter/internal/oasparser/envoyconf/constants.go b/adapter/internal/oasparser/envoyconf/constants.go index 1798d6d19a..c92501c054 100644 --- a/adapter/internal/oasparser/envoyconf/constants.go +++ b/adapter/internal/oasparser/envoyconf/constants.go @@ -29,6 +29,7 @@ const ( const ( httpConManagerStartPrefix string = "ingress_http" extAuthzPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute" + extProcPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute" luaPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute" corsFilterName string = "type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors" localRateLimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit" diff --git a/adapter/internal/oasparser/envoyconf/http_filters.go b/adapter/internal/oasparser/envoyconf/http_filters.go index 4faad45e96..06867d6c76 100644 --- a/adapter/internal/oasparser/envoyconf/http_filters.go +++ b/adapter/internal/oasparser/envoyconf/http_filters.go @@ -28,6 +28,7 @@ import ( envoy_config_ratelimit_v3 "github.com/envoyproxy/go-control-plane/envoy/config/ratelimit/v3" cors_filter_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" ext_authv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + ext_process "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" luav3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" ratelimit "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ratelimit/v3" routerv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" @@ -47,9 +48,14 @@ import ( "github.com/golang/protobuf/ptypes/any" ) + +// HTTPExternalProcessor HTTP filter +const HTTPExternalProcessor = "envoy.filters.http.ext_proc" + // getHTTPFilters generates httpFilter configuration func getHTTPFilters(globalLuaScript string) []*hcmv3.HttpFilter { extAuth := getExtAuthzHTTPFilter() + extProcessor := getExtProcessHTTPFilter() router := getRouterHTTPFilter() luaLocal := getLuaFilter(LuaLocal, ` function envoy_on_request(request_handle) @@ -64,6 +70,7 @@ end`) extAuth, luaLocal, luaGlobal, + extProcessor, } conf := config.ReadConfigs() if conf.Envoy.RateLimit.Enabled { @@ -190,6 +197,43 @@ func getRateLimitFilter() *hcmv3.HttpFilter { return &rlFilter } +// getExtProcessHTTPFilter gets ExtAauthz http filter. +func getExtProcessHTTPFilter() *hcmv3.HttpFilter { + // conf := config.ReadConfigs() + externalProcessor := &ext_process.ExternalProcessor{ + GrpcService: &corev3.GrpcService{ + TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ + ClusterName: extAuthzClusterName, + }, + }, + }, + ProcessingMode: &ext_process.ProcessingMode{ + ResponseBodyMode: ext_process.ProcessingMode_BUFFERED, + RequestHeaderMode: ext_process.ProcessingMode_SKIP, + ResponseHeaderMode: ext_process.ProcessingMode_SKIP, + }, + MetadataOptions: &ext_process.MetadataOptions{ + ForwardingNamespaces: &ext_process.MetadataOptions_MetadataNamespaces{ + Untyped: []string{"envoy.filters.http.ext_authz", "envoy.filters.http.ext_proc"}, + }, + }, + RequestAttributes: []string{"xds.route_metadata"}, + ResponseAttributes: []string{"xds.route_metadata"}, + } + ext, err2 := anypb.New(externalProcessor) + if err2 != nil { + logger.LoggerOasparser.Error(err2) + } + extProcessFilter := hcmv3.HttpFilter{ + Name: HTTPExternalProcessor, + ConfigType: &hcmv3.HttpFilter_TypedConfig{ + TypedConfig: ext, + }, + } + return &extProcessFilter +} + // getExtAuthzHTTPFilter gets ExtAauthz http filter. func getExtAuthzHTTPFilter() *hcmv3.HttpFilter { conf := config.ReadConfigs() diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index 12b87c0a25..797193e7ad 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -27,6 +27,7 @@ import ( corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" envoy_type_matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" @@ -40,6 +41,11 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + v35 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" +) + +const( + authzNamespace = "envoy.filters.http.ext_authz" ) // Constants for Rate Limiting @@ -97,7 +103,7 @@ func generateRouteMatch(routeRegex string) *routev3.RouteMatch { return match } -func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string) (action *routev3.Route_Route) { +func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string, isSubscriptionBasedAIRatelimitEnabled bool, isBackendBasedAIRatelimitEnabled bool, descriptorValueForBackendBasedAIRatelimit string) (action *routev3.Route_Route) { action = &routev3.Route_Route{ Route: &routev3.RouteAction{ HostRewriteSpecifier: &routev3.RouteAction_AutoHostRewrite{ @@ -128,6 +134,9 @@ func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, rate if ratelimitCriteria != nil && ratelimitCriteria.level != "" { action.Route.RateLimits = generateRateLimitPolicy(ratelimitCriteria) } + if isBackendBasedAIRatelimitEnabled { + action.Route.RateLimits = append(action.Route.RateLimits, generateBackendBasedAIRatelimit(descriptorValueForBackendBasedAIRatelimit)...) + } // Add request mirroring configurations if mirrorClusterNames != nil && len(mirrorClusterNames) > 0 { @@ -182,6 +191,136 @@ func mapStatusCodeToEnum(statusCode int) int { return -1 } } +const ( + // DescriptorKeyForAIRequestTokenCount is the descriptor key for AI request token count ratelimit + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + // DescriptorKeyForAIResponseTokenCount is the descriptor key for AI response token count ratelimit + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + // DescriptorKeyForAITotalTokenCount is the descriptor key for AI total token count ratelimit + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + // DescriptorKeyForAIRequestCount is the descriptor key for AI request count ratelimit + DescriptorKeyForAIRequestCount = "airequestcount" +) + +func generateBackendBasedAIRatelimit(descValue string) []*routev3.RateLimit { + rateLimitForRequestTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIRequestTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForResponseTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIResponseTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForTotalTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAITotalTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForRequestCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIRequestCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount, &rateLimitForTotalTokenCount} +} + + +func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { + rateLimitForRequestTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestTokenCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + rateLimitForResponseTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIResponseTokenCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + rateLimitForRequestCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount} +} func generateRateLimitPolicy(ratelimitCriteria *ratelimitCriteria) []*routev3.RateLimit { environmentValue := ratelimitCriteria.environment @@ -526,9 +665,21 @@ func generateFilterConfigToSkipEnforcer() map[string]*anypb.Any { TypeUrl: extAuthzPerRouteName, Value: data, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } return map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, } } diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 85f673c38a..674f1c8086 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -39,6 +39,7 @@ import ( routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" cors_filter_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" lua "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" upstreams "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" @@ -859,13 +860,25 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } corsFilter, _ := anypb.New(corsPolicy) - perRouteFilterConfigs := map[string]*any.Any{ wellknown.HTTPExternalAuthorization: extAuthzFilter, LuaLocal: luaFilter, wellknown.CORS: corsFilter, } - + if !resource.GetEnableBackendBasedAIRatelimit() && !resource.GetEnableSubscriptionBasedAIRatelimit() { + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + perRouteFilterConfigs[HTTPExternalProcessor] = filterExtProc + } + logger.LoggerOasparser.Debugf("adding route : %s for API : %s", resourcePath, title) rateLimitPolicyLevel := "" @@ -916,6 +929,34 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } } routeConfig := resource.GetEndpoints().Config + metaData := &corev3.Metadata{} + if resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit() { + metaData = &corev3.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "EnableBackendBasedAIRatelimit": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: fmt.Sprintf("%t", resource.GetEnableBackendBasedAIRatelimit()), + }, + }, + "EnableSubscriptionBasedAIRatelimit": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: fmt.Sprintf("%t", resource.GetEnableSubscriptionBasedAIRatelimit()), + }, + }, + "BackendBasedAIRatelimitDescriptorValue": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: resource.GetBackendBasedAIRatelimitDescriptorValue(), + }, + }, + }, + }, + }, + } + } else { + metaData = nil + } if resource.HasPolicies() { logger.LoggerOasparser.Debug("Start creating routes for resource with policies") operations := resource.GetOperations() @@ -1050,12 +1091,12 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error metadataValue := operation.GetMethod() + "_to_" + newMethod match2.DynamicMetadata = generateMetadataMatcherForInternalRoutes(metadataValue) - action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) - action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) + action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) + action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) // Create route1 for current method. // Do not add policies to route config. Send via enforcer - route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, nil, decorator, perRouteFilterConfigs, + route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // Create route2 for new method. @@ -1066,7 +1107,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action2.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } configToSkipEnforcer := generateFilterConfigToSkipEnforcer() - route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, nil, decorator, configToSkipEnforcer, + route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, metaData, decorator, configToSkipEnforcer, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route1) @@ -1074,7 +1115,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else { var action *routev3.Route_Route if requestRedirectAction == nil { - action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) + action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) } logger.LoggerOasparser.Debug("Creating routes for resource with policies", resourcePath, operation.GetMethod()) // create route for current method. Add policies to route config. Send via enforcer @@ -1086,7 +1127,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else if requestRedirectAction == nil { action.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } - route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, nil, decorator, perRouteFilterConfigs, + route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route) } @@ -1100,11 +1141,11 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } match := generateRouteMatch(routePath) match.Headers = generateHTTPMethodMatcher(methodRegex, clusterName) - action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil) + action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) - - route := generateRouteConfig(xWso2Basepath, match, action, nil, nil, decorator, perRouteFilterConfigs, + + route := generateRouteConfig(xWso2Basepath, match, action, nil, metaData, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) } @@ -1222,6 +1263,18 @@ func CreateAPIDefinitionRoute(basePath string, vHost string, methods []string, i }, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = routev3.Route{ Name: apiDefinitionQueryParam, Match: match, @@ -1230,6 +1283,7 @@ func CreateAPIDefinitionRoute(basePath string, vHost string, methods []string, i Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return &router @@ -1303,6 +1357,18 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v }, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = &routev3.Route{ Name: endpoint, //Categorize routes with same base path Match: match, @@ -1311,6 +1377,7 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return router @@ -1347,6 +1414,17 @@ func CreateHealthEndpoint() *routev3.Route { Value: data, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } router = routev3.Route{ Name: healthPath, //Categorize routes with same base path Match: match, @@ -1364,6 +1442,7 @@ func CreateHealthEndpoint() *routev3.Route { Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return &router @@ -1388,6 +1467,18 @@ func CreateReadyEndpoint() *routev3.Route { Operation: readyPath, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = routev3.Route{ Name: readyPath, //Categorize routes with same base path Match: match, @@ -1400,6 +1491,9 @@ func CreateReadyEndpoint() *routev3.Route { }, Metadata: nil, Decorator: decorator, + TypedPerFilterConfig: map[string]*any.Any{ + HTTPExternalProcessor : filterExtProc, + }, } return &router } diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 56f4558202..9dabb2f622 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -501,7 +501,7 @@ func (adapterInternalAPI *AdapterInternalAPI) Validate() error { // SetInfoHTTPRouteCR populates resources and endpoints of adapterInternalAPI. httpRoute.Spec.Rules.Matches // are used to create resources and httpRoute.Spec.Rules.BackendRefs are used to create EndpointClusters. -func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams) error { +func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams, isAiSubscriptionRatelimitEnabled bool, ruleIdxToAiRatelimitPolicyMapping map[int]*dpv1alpha3.AIRateLimitPolicy) error { var resources []*Resource outputAuthScheme := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.AuthSchemes))) outputAPIPolicy := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.APIPolicies))) @@ -523,7 +523,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap ratelimitPolicy = *outputRatelimitPolicy } - for _, rule := range httpRoute.Spec.Rules { + for ruleID, rule := range httpRoute.Spec.Rules { var endPoints []Endpoint var policies = OperationPolicies{} var circuitBreaker *dpv1alpha2.CircuitBreaker @@ -545,6 +545,16 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap var securityConfig []EndpointSecurity var mirrorEndpointClusters []*EndpointCluster + enableBackendBasedAIRatelimit := false + descriptorValue := "" + if aiRatelimitPolicy, exists := ruleIdxToAiRatelimitPolicyMapping[ruleID]; exists { + loggers.LoggerAPI.Infof("Found AI ratelimit mapping for ruleId: %d, related api: %s", ruleID, adapterInternalAPI.UUID) + enableBackendBasedAIRatelimit = true + descriptorValue = prepareAIRatelimitIdentifier(adapterInternalAPI.OrganizationID, utils.NamespacedName(aiRatelimitPolicy), &aiRatelimitPolicy.Spec) + } else { + loggers.LoggerAPI.Infof("Could not find AIratelimit for ruleId: %d, len of map: %d, related api: %s", ruleID, len(ruleIdxToAiRatelimitPolicyMapping), adapterInternalAPI.UUID) + } + backendBasePath := "" for _, backend := range rule.BackendRefs { backendName := types.NamespacedName{ @@ -901,12 +911,17 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap operations := getAllowedOperations(match.Method, policies, apiAuth, parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes, mirrorEndpointClusters) - resource := &Resource{path: resourcePath, + + resource := &Resource{ + path: resourcePath, methods: operations, pathMatchType: *match.Path.Type, hasPolicies: true, iD: uuid.New().String(), hasRequestRedirectFilter: hasRequestRedirectPolicy, + enableSubscriptionBasedAIRatelimit: isAiSubscriptionRatelimitEnabled, + enableBackendBasedAIRatelimit: enableBackendBasedAIRatelimit, + backendBasedAIRatelimitDescriptorValue: descriptorValue, } resource.endpoints = &EndpointCluster{ @@ -1364,3 +1379,11 @@ func CreateDummyAdapterInternalAPIForTests(title, version, basePath string, reso resources: resources, } } + +func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { + targetNamespace := string(namespacedName.Namespace) + if spec.TargetRef.Namespace != nil && string(*spec.TargetRef.Namespace) != "" { + targetNamespace = string(*spec.TargetRef.Namespace) + } + return fmt.Sprintf("%s-%s-%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name), targetNamespace, string(spec.TargetRef.Name)) +} diff --git a/adapter/internal/oasparser/model/resource.go b/adapter/internal/oasparser/model/resource.go index 9fafd7f875..a953e3a850 100644 --- a/adapter/internal/oasparser/model/resource.go +++ b/adapter/internal/oasparser/model/resource.go @@ -36,15 +36,18 @@ import ( // These values are populated from extensions/properties // mentioned under pathItem. type Resource struct { - path string - pathMatchType gwapiv1.PathMatchType - methods []*Operation - iD string - endpoints *EndpointCluster - endpointSecurity []*EndpointSecurity - vendorExtensions map[string]interface{} - hasPolicies bool - hasRequestRedirectFilter bool + path string + pathMatchType gwapiv1.PathMatchType + methods []*Operation + iD string + endpoints *EndpointCluster + endpointSecurity []*EndpointSecurity + vendorExtensions map[string]interface{} + hasPolicies bool + hasRequestRedirectFilter bool + enableSubscriptionBasedAIRatelimit bool + enableBackendBasedAIRatelimit bool + backendBasedAIRatelimitDescriptorValue string } // GetEndpointSecurity returns the endpoint security object of a given resource. @@ -189,3 +192,18 @@ func SortResources(resources []*Resource) []*Resource { sort.Sort(byPath(resources)) return resources } + +// GetEnableSubscriptionBasedAIRatelimit returns the value of enableSubscriptionBasedAIRatelimit. +func (resource *Resource) GetEnableSubscriptionBasedAIRatelimit() bool { + return resource.enableSubscriptionBasedAIRatelimit +} + +// GetEnableBackendBasedAIRatelimit returns the value of enableBackendBasedAIRatelimit. +func (resource *Resource) GetEnableBackendBasedAIRatelimit() bool { + return resource.enableBackendBasedAIRatelimit +} + +// GetBackendBasedAIRatelimitDescriptorValue returns the value of backendBasedAIRatelimitDescriptorValue. +func (resource *Resource) GetBackendBasedAIRatelimitDescriptorValue() string { + return resource.backendBasedAIRatelimitDescriptorValue +} \ No newline at end of file diff --git a/adapter/internal/operator/PROJECT b/adapter/internal/operator/PROJECT index 1c9a275a47..70447f6ad7 100644 --- a/adapter/internal/operator/PROJECT +++ b/adapter/internal/operator/PROJECT @@ -155,4 +155,13 @@ resources: kind: GQLRoute path: github.com/wso2/apk/adapter/internal/operator/apis/dp/v1alpha2 version: v1alpha2 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: wso2.com + group: dp + kind: AIRateLimitPolicy + path: github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3 + version: v1alpha3 version: "3" diff --git a/adapter/internal/operator/config/crd/kustomization.yaml b/adapter/internal/operator/config/crd/kustomization.yaml index f2eb17419e..5837a1fc4e 100644 --- a/adapter/internal/operator/config/crd/kustomization.yaml +++ b/adapter/internal/operator/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/cp.wso2.com_subscriptions.yaml - bases/dp.wso2.com_scopes.yaml - bases/dp.wso2.com_ratelimitpolicies.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml - bases/dp.wso2.com_backends.yaml - bases/dp.wso2.com_interceptorservices.yaml - bases/dp.wso2.com_jwtissuers.yaml diff --git a/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml b/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..2d28892f72 --- /dev/null +++ b/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: airatelimitpolicies.dp.wso2.com diff --git a/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml b/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b2cde3fe7d --- /dev/null +++ b/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: airatelimitpolicies.dp.wso2.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml new file mode 100644 index 0000000000..c6afbc67c6 --- /dev/null +++ b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-editor-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml new file mode 100644 index 0000000000..5fe04d1924 --- /dev/null +++ b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-viewer-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - get + - list + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/adapter/internal/operator/constants/constants.go b/adapter/internal/operator/constants/constants.go index 872c3491e3..cbafee9184 100644 --- a/adapter/internal/operator/constants/constants.go +++ b/adapter/internal/operator/constants/constants.go @@ -45,10 +45,11 @@ const ( KindAPI = "API" KindService = "Service" //TODO(amali) remove this after fixing the issue in https://github.com/wso2/apk/issues/383 - KindResource = "Resource" - KindScope = "Scope" - KindBackend = "Backend" - KindGateway = "Gateway" + KindResource = "Resource" + KindScope = "Scope" + KindBackend = "Backend" + KindGateway = "Gateway" + KindSubscription = "Subscription" ) // Env types diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index c614eaa0e3..30ce6a4918 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -58,8 +58,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" k8client "sigs.k8s.io/controller-runtime/pkg/client" + cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" - "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" corev1 "k8s.io/api/core/v1" @@ -82,20 +82,24 @@ const ( // apiAPIPolicyIndex Index for API level apipolicies apiAPIPolicyIndex = "apiAPIPolicyIndex" // apiAPIPolicyResourceIndex Index for resource level apipolicies - apiAPIPolicyResourceIndex = "apiAPIPolicyResourceIndex" - serviceHTTPRouteIndex = "serviceHTTPRouteIndex" - httprouteScopeIndex = "httprouteScopeIndex" - gqlRouteScopeIndex = "gqlRouteScopeIndex" - configMapBackend = "configMapBackend" - configMapAPIDefinition = "configMapAPIDefinition" - secretBackend = "secretBackend" - configMapAuthentication = "configMapAuthentication" - secretAuthentication = "secretAuthentication" - backendHTTPRouteIndex = "backendHTTPRouteIndex" - backendGQLRouteIndex = "backendGQLRouteIndex" - interceptorServiceAPIPolicyIndex = "interceptorServiceAPIPolicyIndex" - backendInterceptorServiceIndex = "backendInterceptorServiceIndex" - backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" + apiAPIPolicyResourceIndex = "apiAPIPolicyResourceIndex" + serviceHTTPRouteIndex = "serviceHTTPRouteIndex" + httprouteScopeIndex = "httprouteScopeIndex" + gqlRouteScopeIndex = "gqlRouteScopeIndex" + configMapBackend = "configMapBackend" + configMapAPIDefinition = "configMapAPIDefinition" + secretBackend = "secretBackend" + configMapAuthentication = "configMapAuthentication" + secretAuthentication = "secretAuthentication" + backendHTTPRouteIndex = "backendHTTPRouteIndex" + backendGQLRouteIndex = "backendGQLRouteIndex" + interceptorServiceAPIPolicyIndex = "interceptorServiceAPIPolicyIndex" + backendInterceptorServiceIndex = "backendInterceptorServiceIndex" + backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" + aiRatelimitPolicyToBackendIndex = "aiRatelimitPolicyToBackendIndex" + aiRatelimitPolicyToSubscriptionIndex = "aiRatelimitPolicyToSubscriptionIndex" + subscriptionToAPIIndex = "subscriptionToAPIIndex" + apiToSubscriptionIndex = "apiToSubscriptionIndex" aiProviderAPIPolicyIndex = "aiProviderAPIPolicyIndex" ) @@ -223,6 +227,17 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2615, logging.BLOCKER, "Error watching AIPolicy resources: %v", err)) return err } + if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForAIRatelimitPolicy), + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching AIRatelimitPolicy resources: %v", err)) + return err + } + + if err := c.Watch(source.Kind(mgr.GetCache(), &cpv1alpha2.Subscription{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForSubscription), + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching Subscription resources: %v", err)) + return err + } loggers.LoggerAPKOperator.Info("API Controller successfully started. Watching API Objects....") go apiReconciler.handleStatus() @@ -251,6 +266,9 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies/status,verbs=get;update;patch // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies/finalizers,verbs=update +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -382,6 +400,7 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 apiRef.String(), namespace, string(api.ObjectMeta.UID), "api definition file not found") } } + apiReconciler.resolveAiSubscriptionRatelimitPolicies(ctx, apiState) if len(apiState.Authentications) > 0 { if apiState.MutualSSL, err = apiReconciler.resolveAuthentications(ctx, apiState.Authentications); err != nil { return nil, fmt.Errorf("error while resolving authentication %v in namespace: %s was not found. %s", @@ -836,6 +855,29 @@ func (apiReconciler *APIReconciler) getAPIPolicyChildrenRefs(ctx context.Context return interceptorServices, backendJWTs, subscriptionValidation, aiProvider, nil } +func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx context.Context, apiState *synchronizer.APIState) { + apiState.IsAiSubscriptionRatelimitEnabled = false + subscriptionList := &cpv1alpha2.SubscriptionList{} + if err := apiReconciler.client.List(ctx, subscriptionList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(subscriptionToAPIIndex, utils.GetSubscriptionToAPIIndexID(apiState.APIDefinition.Spec.APIName, apiState.APIDefinition.Spec.APIVersion)), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated subscription found for API: %s", utils.NamespacedName(apiState.APIDefinition)) + return + } + for _, subscription := range subscriptionList.Items { + aiRatelimitPolicyList := &dpv1alpha3.AIRateLimitPolicyList{} + if err := apiReconciler.client.List(ctx, aiRatelimitPolicyList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToSubscriptionIndex, utils.NamespacedName(&subscription).String()), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) + return + } + if len(aiRatelimitPolicyList.Items) > 0 { + apiState.IsAiSubscriptionRatelimitEnabled = true + } + } +} + func (apiReconciler *APIReconciler) resolveAuthentications(ctx context.Context, authentications map[string]dpv1alpha2.Authentication) (*dpv1alpha2.MutualSSL, error) { resolvedMutualSSL := dpv1alpha2.MutualSSL{} @@ -855,12 +897,44 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte // Resolve backends in HTTPRoute httpRoute := httpRouteState.HTTPRouteCombined - for _, rule := range httpRoute.Spec.Rules { + // httpRouteState.AiRatelimit_HttpRouteRulesMapping = aiRatelimitPolicy_routeRulematching + // httpRouteState.AiRatelimitPolicyMapping = aiRatelimitPolicyMapping + ruleIdxToAiRatelimitPolicyMapping := make(map[int]*dpv1alpha3.AIRateLimitPolicy) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = ruleIdxToAiRatelimitPolicyMapping + for id, rule := range httpRoute.Spec.Rules { for _, backend := range rule.BackendRefs { backendNamespacedName := types.NamespacedName{ Name: string(backend.Name), Namespace: utils.GetNamespace(backend.Namespace, httpRoute.Namespace), } + aiRLPolicyList := &dpv1alpha3.AIRateLimitPolicyList{} + if err := apiReconciler.client.List(ctx, aiRLPolicyList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToBackendIndex, backendNamespacedName.String()), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated AI ratelimit policy found for : %s", backendNamespacedName.String()) + } else { + for _, aiRLPolicy := range aiRLPolicyList.Items { + loggers.LoggerAPKOperator.Infof("Adding mapping for ruleid: %d to aiRLPolicy: %s", id, utils.NamespacedName(&aiRLPolicy)) + ruleIdxToAiRatelimitPolicyMapping[id] = &aiRLPolicy + // aiRlPolicyNN := utils.NamespacedName(&aiRLPolicy) + // if ruleList, exists := aiRatelimitPolicy_routeRulematching[aiRlPolicyNN]; !exists { + // ruleListNew := make([]*gwapiv1.HTTPRouteRule, 0) + // ruleListNew = append(ruleListNew, &rule) + // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleListNew + // } else { + // ruleList = append(ruleList, &rule) + // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleList + // } + // if aiRLList, exists := aiRatelimitPolicyMapping[aiRlPolicyNN]; !exists { + // aiRlListNew := make([]*v1alpha3.AIRateLimitPolicy, 0) + // aiRlListNew = append(aiRlListNew, &aiRLPolicy) + // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRlListNew + // } else { + // aiRLList = append(aiRLList, &aiRLPolicy) + // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRLList + // } + } + } if _, exists := backendMapping[backendNamespacedName.String()]; !exists { resolvedBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, backendNamespacedName, &api) if resolvedBackend != nil { @@ -869,6 +943,7 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte return nil, fmt.Errorf("unable to find backend %s", backendNamespacedName.String()) } } + } for _, filter := range rule.Filters { @@ -948,6 +1023,22 @@ func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForSecret(ctx co return requests } +func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAIRatelimitPolicy(ctx context.Context, obj k8client.Object) []reconcile.Request { + requests := apiReconciler.getAPIsForAIRatelimitPolicy(ctx, obj) + if len(requests) > 0 { + apiReconciler.handleOwnerReference(ctx, obj, &requests) + } + return requests +} + +func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { + requests := apiReconciler.getAPIsForSubscription(ctx, obj) + // if len(requests) > 0 { + // apiReconciler.handleOwnerReference(ctx, obj, &requests) + // } + return requests +} + func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAuthentication(ctx context.Context, obj k8client.Object) []reconcile.Request { requests := apiReconciler.getAPIsForAuthentication(ctx, obj) if len(requests) > 0 { @@ -1385,6 +1476,60 @@ func (apiReconciler *APIReconciler) getAPIsForSecret(ctx context.Context, obj k8 return requests } +// getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected +// in AIRatelimitPolicy resources. +func (apiReconciler *APIReconciler) getAPIsForAIRatelimitPolicy(ctx context.Context, obj k8client.Object) []reconcile.Request { + aiRatelimitPolicy, ok := obj.(*dpv1alpha3.AIRateLimitPolicy) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) + return []reconcile.Request{} + } + + if aiRatelimitPolicy.Spec.TargetRef.Kind == constants.KindBackend { + backend := &dpv1alpha1.Backend{} + namespacedName := types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()), + } + + if err := apiReconciler.client.Get(ctx, namespacedName, backend); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2621, logging.MINOR, "Unable to find associated Backend for AIratelimitPolicy targetref: %s", namespacedName.String())) + return []reconcile.Request{} + } + return apiReconciler.getAPIsForBackend(ctx, backend) + } + return []reconcile.Request{} +} + +// getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected +// in subscription resources. +func (apiReconciler *APIReconciler) getAPIsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { + subscription, ok := obj.(*cpv1alpha2.Subscription) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) + return []reconcile.Request{} + } + apiList := &dpv1alpha2.APIList{} + if err := apiReconciler.client.List(ctx, apiList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(apiToSubscriptionIndex, utils.GetSubscriptionToAPIIndexID(subscription.Spec.API.Name, subscription.Spec.API.Version)), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2649, logging.CRITICAL, "Unable to find associated APIs for subscription: %s, error: %v", utils.NamespacedName(subscription).String(), err.Error())) + return []reconcile.Request{} + } + requests := []reconcile.Request{} + for _, api := range apiList.Items { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: api.Name, + Namespace: api.Namespace}, + } + requests = append(requests, req) + loggers.LoggerAPKOperator.Infof("Adding reconcile request for API: %s/%s with API UUID: %v due to change in subscription: %v", api.Namespace, api.Name, + string(api.ObjectMeta.UID), utils.NamespacedName(subscription).String()) + } + return requests +} + // getAPIForAuthentication triggers the API controller reconcile method based on the changes detected // from Authentication objects. If the changes are done for an API stored in the Operator Data store, // a new reconcile event will be created and added to the reconcile event queue. @@ -1941,6 +2086,60 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } + // AIRatelimitPolicy to Backend indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToBackendIndex, + func(rawObj k8client.Object) []string { + aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) + var backends []string + namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) + backends = append(backends, types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: namespace, + }.String()) + return backends + }); err != nil { + return err + } + + // AIRatelimitPolicy to Subscription indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToSubscriptionIndex, + func(rawObj k8client.Object) []string { + aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) + var subscriptions []string + namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) + subscriptions = append(subscriptions, types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: namespace, + }.String()) + return subscriptions + }); err != nil { + return err + } + + // Subscription to API indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha2.Subscription{}, subscriptionToAPIIndex, + func(rawObj k8client.Object) []string { + subscription := rawObj.(*cpv1alpha2.Subscription) + var subscriptions []string + subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", subscription.Spec.API.Name, subscription.Spec.API.Version) + subscriptions = append(subscriptions, subscriptionIdentifierForIndex) + return subscriptions + }); err != nil { + return err + } + + // API to Subscription indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.API{}, apiToSubscriptionIndex, + func(rawObj k8client.Object) []string { + api := rawObj.(*dpv1alpha2.API) + var apis []string + subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", api.Spec.APIName, api.Spec.APIVersion) + apis = append(apis, subscriptionIdentifierForIndex) + return apis + }); err != nil { + return err + } + // authentication to API indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.Authentication{}, apiAuthenticationIndex, func(rawObj k8client.Object) []string { @@ -2769,7 +2968,7 @@ func geSandVhost(apiState *synchronizer.APIState) string { } func prepareSecuritySchemeForCP(apiState *synchronizer.APIState) ([]string, string, string) { - var pickedAuth *v1alpha2.Authentication + var pickedAuth *dpv1alpha2.Authentication authHeader := "Authorization" apiKeyHeader := "ApiKey" for _, auth := range apiState.Authentications { @@ -2777,7 +2976,7 @@ func prepareSecuritySchemeForCP(apiState *synchronizer.APIState) ([]string, stri break } if pickedAuth != nil { - var authSpec *v1alpha2.AuthSpec + var authSpec *dpv1alpha2.AuthSpec if pickedAuth.Spec.Override != nil { authSpec = pickedAuth.Spec.Override } else { diff --git a/adapter/internal/operator/controllers/dp/suite_test.go b/adapter/internal/operator/controllers/dp/suite_test.go index 017df70bcb..3dbdd04c94 100644 --- a/adapter/internal/operator/controllers/dp/suite_test.go +++ b/adapter/internal/operator/controllers/dp/suite_test.go @@ -33,6 +33,7 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" //+kubebuilder:scaffold:imports ) @@ -70,6 +71,9 @@ var _ = BeforeSuite(func() { err = dpv1alpha2.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = dpv1alpha3.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index afeb516220..ece0e47e1a 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -50,6 +50,7 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" //+kubebuilder:scaffold:imports ) @@ -69,6 +70,8 @@ func init() { utilruntime.Must(dpv1alpha2.AddToScheme(scheme)) utilruntime.Must(dpv1alpha3.AddToScheme(scheme)) + + utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/adapter/internal/operator/synchronizer/api_state.go b/adapter/internal/operator/synchronizer/api_state.go index 3a1e6b3dea..8536cd62af 100644 --- a/adapter/internal/operator/synchronizer/api_state.go +++ b/adapter/internal/operator/synchronizer/api_state.go @@ -45,6 +45,7 @@ type APIState struct { APIDefinitionFile []byte SubscriptionValidation bool MutualSSL *v1alpha2.MutualSSL + IsAiSubscriptionRatelimitEnabled bool } // HTTPRouteState holds the state of the deployed httpRoutes. This state is compared with @@ -55,6 +56,7 @@ type HTTPRouteState struct { HTTPRoutePartitions map[string]*gwapiv1.HTTPRoute BackendMapping map[string]*v1alpha2.ResolvedBackend Scopes map[string]v1alpha1.Scope + RuleIdxToAiRatelimitPolicyMapping map[int]*v1alpha3.AIRateLimitPolicy } // GQLRouteState holds the state of the deployed gqlRoutes. This state is compared with diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 0c861d315b..1ecb67f1c2 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -90,7 +90,28 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced "Production"); routesUpdated { updated = true events = append(events, routeEvents...) + } else { + // Check whether AIRatelimitPolicy is updated + for key, aiRl := range apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { + if cachedAIRl, exists := cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { + if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { + loggers.LoggerAPI.Infof("Returning true * %s %s %d %d", utils.NamespacedName(cachedAIRl).String(), utils.NamespacedName(aiRl).String(), cachedAIRl.Generation, aiRl.Generation) + updated = true + break + } + } else { + loggers.LoggerAPI.Info("Returning true&&") + updated = true + break + } + } + if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { + loggers.LoggerAPI.Info("Returning true ***") + updated = true + } } + cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping = apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping + } else { if cachedAPI.ProdHTTPRoute != nil { updated = true @@ -98,6 +119,7 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } cachedAPI.ProdHTTPRoute = nil } + if apiState.ProdGQLRoute != nil { if cachedAPI.ProdGQLRoute == nil { cachedAPI.ProdGQLRoute = apiState.ProdGQLRoute @@ -123,7 +145,27 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } else if routeEvents, routesUpdated := updateHTTPRoute(apiState.SandHTTPRoute, cachedAPI.SandHTTPRoute, "Sandbox"); routesUpdated { updated = true events = append(events, routeEvents...) + } else { + // Check whether AIRatelimitPolicy is updated + for key, aiRl := range apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { + if cachedAIRl, exists := cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { + if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { + loggers.LoggerAPI.Info("Returning true") + updated = true + break + } + } else { + loggers.LoggerAPI.Info("Returning true") + updated = true + break + } + } + if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { + loggers.LoggerAPI.Info("Returning true") + updated = true + } } + cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping = apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping } else { if cachedAPI.SandHTTPRoute != nil { updated = true diff --git a/adapter/internal/operator/synchronizer/rest_api.go b/adapter/internal/operator/synchronizer/rest_api.go index b94ce90ae8..8c82aaa486 100644 --- a/adapter/internal/operator/synchronizer/rest_api.go +++ b/adapter/internal/operator/synchronizer/rest_api.go @@ -78,7 +78,7 @@ func UpdateInternalMapsFromHTTPRoute(apiState APIState, httpRoute *HTTPRouteStat } // generateAdapterInternalAPI this will populate a AdapterInternalAPI representation for an HTTPRoute -func generateAdapterInternalAPI(apiState APIState, httpRoute *HTTPRouteState, envType string) (*model.AdapterInternalAPI, error) { +func generateAdapterInternalAPI(apiState APIState, httpRouteState *HTTPRouteState, envType string) (*model.AdapterInternalAPI, error) { var adapterInternalAPI model.AdapterInternalAPI adapterInternalAPI.SetIsDefaultVersion(apiState.APIDefinition.Spec.IsDefaultVersion) adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) @@ -98,16 +98,16 @@ func generateAdapterInternalAPI(apiState APIState, httpRoute *HTTPRouteState, en resourceParams := model.ResourceParams{ AuthSchemes: apiState.Authentications, ResourceAuthSchemes: apiState.ResourceAuthentications, - BackendMapping: httpRoute.BackendMapping, + BackendMapping: httpRouteState.BackendMapping, APIPolicies: apiState.APIPolicies, ResourceAPIPolicies: apiState.ResourceAPIPolicies, - ResourceScopes: httpRoute.Scopes, + ResourceScopes: httpRouteState.Scopes, InterceptorServiceMapping: apiState.InterceptorServiceMapping, BackendJWTMapping: apiState.BackendJWTMapping, RateLimitPolicies: apiState.RateLimitPolicies, ResourceRateLimitPolicies: apiState.ResourceRateLimitPolicies, } - if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRoute.HTTPRouteCombined, resourceParams); err != nil { + if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRouteState.HTTPRouteCombined, resourceParams, apiState.IsAiSubscriptionRatelimitEnabled, httpRouteState.RuleIdxToAiRatelimitPolicyMapping); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2631, logging.MAJOR, "Error setting HttpRoute CR info to adapterInternalAPI. %v", err)) return nil, err } diff --git a/adapter/internal/operator/utils/utils.go b/adapter/internal/operator/utils/utils.go index 2e8b3e99a1..b3ecd1275c 100644 --- a/adapter/internal/operator/utils/utils.go +++ b/adapter/internal/operator/utils/utils.go @@ -689,3 +689,8 @@ func ContainsString(list []string, target string) bool { } return false } + +// GetSubscriptionToAPIIndexID returns the id which can be used to list subscriptions related to a api. +func GetSubscriptionToAPIIndexID(name string, version string) string { + return fmt.Sprintf("%s_%s", name, version) +} diff --git a/common-controller/go.mod b/common-controller/go.mod index 5e0e92001e..d5707c7a84 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -16,14 +16,14 @@ require ( ) require ( - github.com/envoyproxy/go-control-plane v0.12.0 + github.com/envoyproxy/go-control-plane v0.13.0 github.com/gin-gonic/gin v1.9.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 github.com/wso2/apk/adapter v0.0.0-20231214082511-af2c8b8a19f1 github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 - google.golang.org/grpc v1.62.0 + google.golang.org/grpc v1.65.0 ) replace github.com/wso2/apk/adapter => ../adapter @@ -31,6 +31,7 @@ replace github.com/wso2/apk/adapter => ../adapter replace github.com/wso2/apk/common-go-libs => ../common-go-libs require ( + cel.dev/expr v0.15.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -52,6 +53,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect @@ -63,16 +65,16 @@ require ( github.com/vektah/gqlparser v1.3.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -86,7 +88,7 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect @@ -99,24 +101,22 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/protobuf v1.34.1 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index 44ab8d7761..e3caf33f31 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -19,8 +21,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -29,8 +31,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -45,8 +47,8 @@ github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= -github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= @@ -100,14 +102,11 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -179,6 +178,8 @@ github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNc github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -187,8 +188,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -225,8 +226,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -239,7 +241,6 @@ github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqf github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -257,9 +258,8 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= @@ -268,7 +268,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -277,21 +276,18 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -299,28 +295,21 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -332,7 +321,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -343,26 +331,20 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/common-controller/internal/cache/datastore.go b/common-controller/internal/cache/datastore.go index d9b71480b1..293a3c9e3c 100644 --- a/common-controller/internal/cache/datastore.go +++ b/common-controller/internal/cache/datastore.go @@ -33,6 +33,7 @@ type RatelimitDataStore struct { resolveSubscriptionRatelimitStore map[types.NamespacedName]dpv1alpha3.ResolveSubscriptionRatelimitPolicy customRatelimitStore map[types.NamespacedName]*dpv1alpha1.CustomRateLimitPolicyDef mu sync.Mutex + aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec } // CreateNewOperatorDataStore creates a new RatelimitDataStore. @@ -89,6 +90,18 @@ func (ods *RatelimitDataStore) AddorUpdateCustomRatelimitToStore(rateLimit types ods.customRatelimitStore[rateLimit] = &customRateLimitPolicy } +// AddorUpdateAIRatelimitToStore adds a new ratelimit to the RatelimitDataStore. +func (ods *RatelimitDataStore) AddorUpdateAIRatelimitToStore(rateLimit types.NamespacedName, + aiRatelimitSpec dpv1alpha3.AIRateLimitPolicySpec) { + ods.mu.Lock() + defer ods.mu.Unlock() + logger.Infof("Adding/Updating AI ratelimit spec to cache") + if ods.aiRatelimitPolicySpecs == nil { + ods.aiRatelimitPolicySpecs = make(map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) + } + ods.aiRatelimitPolicySpecs[rateLimit] = &aiRatelimitSpec +} + // GetResolveRatelimitPolicy get cached ratelimit func (ods *RatelimitDataStore) GetResolveRatelimitPolicy(rateLimit types.NamespacedName) ([]dpv1alpha1.ResolveRateLimitAPIPolicy, bool) { var rateLimitPolicy []dpv1alpha1.ResolveRateLimitAPIPolicy @@ -109,6 +122,11 @@ func (ods *RatelimitDataStore) GetCachedCustomRatelimitPolicy(rateLimit types.Na return rateLimitPolicy, false } +// GetAIRatelimitPolicySpecs gets all the AIRatelimitPolicy stored in ods +func (ods *RatelimitDataStore) GetAIRatelimitPolicySpecs() map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec { + return ods.aiRatelimitPolicySpecs +} + // DeleteResolveRatelimitPolicy delete from ratelimit cache func (ods *RatelimitDataStore) DeleteResolveRatelimitPolicy(rateLimit types.NamespacedName) { ods.mu.Lock() @@ -125,6 +143,14 @@ func (ods *RatelimitDataStore) DeleteCachedCustomRatelimitPolicy(rateLimit types delete(ods.customRatelimitStore, rateLimit) } +// DeleteAIRatelimitPolicySpec delete from ratelimit cache +func (ods *RatelimitDataStore) DeleteAIRatelimitPolicySpec(rateLimit types.NamespacedName) { + ods.mu.Lock() + defer ods.mu.Unlock() + logger.Debug("Deleting AI ratelimit from cache") + delete(ods.aiRatelimitPolicySpecs, rateLimit) +} + // NamespacedName generates namespaced name for Kubernetes objects func NamespacedName(obj client.Object) types.NamespacedName { return types.NamespacedName{ diff --git a/common-controller/internal/operator/config/crd/kustomization.yaml b/common-controller/internal/operator/config/crd/kustomization.yaml index 169d615e98..f635c32349 100644 --- a/common-controller/internal/operator/config/crd/kustomization.yaml +++ b/common-controller/internal/operator/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/dp.wso2.com_ratelimitpolicies.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml - bases/dp.wso2.com_apis.yaml - bases/cp.wso2.com_applications.yaml - bases/cp.wso2.com_subscriptions.yaml diff --git a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go new file mode 100644 index 0000000000..afe263ae8a --- /dev/null +++ b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dp + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + // "context" + // "fmt" + // "time" + + // logger "github.com/sirupsen/logrus" + // k8error "k8s.io/apimachinery/pkg/api/errors" + // "k8s.io/apimachinery/pkg/fields" + // "k8s.io/apimachinery/pkg/runtime" + // "k8s.io/apimachinery/pkg/types" + // ctrl "sigs.k8s.io/controller-runtime" + // "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + // "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + // "sigs.k8s.io/controller-runtime/pkg/reconcile" + + // k8client "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/source" + // gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/wso2/apk/adapter/pkg/logging" + cache "github.com/wso2/apk/common-controller/internal/cache" + "github.com/wso2/apk/common-controller/internal/config" + loggers "github.com/wso2/apk/common-controller/internal/loggers" + "github.com/wso2/apk/common-controller/internal/utils" + xds "github.com/wso2/apk/common-controller/internal/xds" + // dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" + // dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + "github.com/wso2/apk/common-go-libs/constants" +) + +// AIRateLimitPolicyReconciler reconciles a AIRateLimitPolicy object +type AIRateLimitPolicyReconciler struct { + client.Client + Scheme *runtime.Scheme + ods *cache.RatelimitDataStore +} + +// NewAIRatelimitController creates a new ratelimitcontroller instance. +func NewAIRatelimitController(mgr manager.Manager, ratelimitStore *cache.RatelimitDataStore) error { + aiRateLimitPolicyReconciler := &AIRateLimitPolicyReconciler{ + Client: mgr.GetClient(), + ods: ratelimitStore, + } + + // ctx := context.Background() + // if err := addIndexes(ctx, mgr); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2612, logging.BLOCKER, "Error adding indexes: %v", err)) + // return err + // } + + c, err := controller.New(constants.AIRatelimitController, mgr, controller.Options{Reconciler: aiRateLimitPolicyReconciler}) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2663, logging.BLOCKER, + "Error creating Ratelimit controller: %v", err.Error())) + return err + } + + conf := config.ReadConfigs() + predicates := []predicate.Predicate{predicate.NewPredicateFuncs(utils.FilterByNamespaces(conf.CommonController.Operator.Namespaces))} + + // if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha2.API{}), + // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForAPI), predicates...); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2611, logging.BLOCKER, + // "Error watching API resources: %v", err)) + // return err + // } + + // if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.HTTPRoute{}), + // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForHTTPRoute), predicates...); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2613, logging.BLOCKER, + // "Error watching HTTPRoute resources: %v", err)) + // return err + // } + + if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), &handler.EnqueueRequestForObject{}, predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2639, logging.BLOCKER, + "Error watching Ratelimit resources: %v", err.Error())) + return err + } + + loggers.LoggerAPKOperator.Debug("RatelimitPolicy Controller successfully started. Watching RatelimitPolicy Objects...") + return nil +} + +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the AIRateLimitPolicy object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.4/pkg/reconcile +func (r *AIRateLimitPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile...") + // TODO(user): your logic here + ratelimitKey := req.NamespacedName + var ratelimitPolicy dpv1alpha3.AIRateLimitPolicy + conf := config.ReadConfigs() + + // Check k8s RatelimitPolicy Availbility + if err := r.Client.Get(ctx, ratelimitKey, &ratelimitPolicy); err != nil { + loggers.LoggerAPKOperator.Errorf("Error retrieving AIRatelimit") + // It could be deletion event. So lets try to delete the related entried from the ods and update xds + r.ods.DeleteAIRatelimitPolicySpec(ratelimitKey) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } else { + loggers.LoggerAPKOperator.Infof("ratelimits found") + r.ods.AddorUpdateAIRatelimitToStore(ratelimitKey, ratelimitPolicy.Spec) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } + loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile..*****.") + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *AIRateLimitPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&dpv1alpha3.AIRateLimitPolicy{}). + Complete(r) +} diff --git a/common-controller/internal/operator/controllers/dp/suite_test.go b/common-controller/internal/operator/controllers/dp/suite_test.go index fd1ec07c35..2cb4598924 100644 --- a/common-controller/internal/operator/controllers/dp/suite_test.go +++ b/common-controller/internal/operator/controllers/dp/suite_test.go @@ -31,6 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" + dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" //+kubebuilder:scaffold:imports ) @@ -65,6 +67,12 @@ var _ = BeforeSuite(func() { err = dpv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = dpv1alpha2.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = dpv1alpha3.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go index f68f7ec23a..ef2ab5eadc 100644 --- a/common-controller/internal/operator/operator.go +++ b/common-controller/internal/operator/operator.go @@ -63,6 +63,7 @@ func init() { utilruntime.Must(gwapiv1.AddToScheme(scheme)) utilruntime.Must(dpv1alpha1.AddToScheme(scheme)) utilruntime.Must(dpv1alpha2.AddToScheme(scheme)) + utilruntime.Must(dpv1alpha3.AddToScheme(scheme)) utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) utilruntime.Must(cpv1alpha3.AddToScheme(scheme)) @@ -164,6 +165,10 @@ func InitOperator(metricsConfig config.Metrics) { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3114, logging.MAJOR, "Error creating JWT Issuer controller, error: %v", err)) } + if err := dpcontrollers.NewAIRatelimitController(mgr, ratelimitStore); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3114, logging.MAJOR, + "Error creating JWT Issuer controller, error: %v", err)) + } config := config.ReadConfigs() if !(config.CommonController.ControlPlane.Enabled && config.CommonController.ControlPlane.Persistence.Type == "DB") { diff --git a/common-controller/internal/xds/ratelimiter_cache.go b/common-controller/internal/xds/ratelimiter_cache.go index 15a191a408..9b9d4bb089 100644 --- a/common-controller/internal/xds/ratelimiter_cache.go +++ b/common-controller/internal/xds/ratelimiter_cache.go @@ -33,20 +33,25 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" "github.com/wso2/apk/common-go-libs/constants" + "k8s.io/apimachinery/pkg/types" ) // Constants relevant to the route related ratelimit configurations const ( - DescriptorKeyForOrg = "org" - OrgMetadataKey = "customorg" - DescriptorKeyForEnvironment = "environment" - DescriptorKeyForPath = "path" - DescriptorKeyForMethod = "method" - DescriptorValueForAPIMethod = "ALL" - DescriptorValueForOperationMethod = ":method" - MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" - MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" - apiDefinitionClusterName = "api_definition_cluster" + DescriptorKeyForOrg = "org" + OrgMetadataKey = "customorg" + DescriptorKeyForEnvironment = "environment" + DescriptorKeyForPath = "path" + DescriptorKeyForMethod = "method" + DescriptorValueForAPIMethod = "ALL" + DescriptorValueForOperationMethod = ":method" + MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" + MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" + apiDefinitionClusterName = "api_definition_cluster" + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + DescriptorKeyForAIRequestCount = "airequestcount" ) const ( @@ -81,6 +86,8 @@ type rateLimitPolicyCache struct { // org -> Custom Rate Limit Configs customRateLimitPolicies map[string]map[string]*rls_config.RateLimitDescriptor + aiRatelimitDescriptors []*rls_config.RateLimitDescriptor + // mutex for API level apiLevelMu sync.RWMutex @@ -282,6 +289,8 @@ func (r *rateLimitPolicyCache) generateRateLimitConfig() *rls_config.RateLimitCo } orgDescriptors = append(orgDescriptors, metadataDescriptors...) + // Add AI ratelimit descriptors + orgDescriptors = append(orgDescriptors, r.aiRatelimitDescriptors...) return &rls_config.RateLimitConfig{ Name: RateLimiterDomain, Domain: RateLimiterDomain, @@ -313,6 +322,61 @@ func (r *rateLimitPolicyCache) AddCustomRateLimitPolicies(customRateLimitPolicy } } +// ProcessAIratelimitPolicySpecsAndUpdateCache process the specs and update the cache +func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRateLimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) + loggers.LoggerAPKOperator.Infof("222222") + for namespacedName, spec := range aiRateLimitPolicySpecs { + logger.Infof("Adding : %s, %s", DescriptorKeyForAIRequestCount, prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec)) + logger.Infof("For airl: %s", namespacedName) + // Add descriptor for RequestTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIRequestTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.RequestTokenCount), + }, + }) + // Add descriptor for ResponseTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIResponseTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.ResponseTokenCount), + }, + }) + // Add descriptor for TotalTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAITotalTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.TotalTokenCount), + }, + }) + // Add descriptor for RequestCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIRequestCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.RequestCount.Unit), + RequestsPerUnit: uint32(spec.Override.RequestCount.RequestsPerUnit), + }, + }) + } + r.aiRatelimitDescriptors = aiRlDescriptors +} + +func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { + targetNamespace := string(namespacedName.Namespace) + if spec.TargetRef.Namespace != nil && string(*spec.TargetRef.Namespace) != "" { + targetNamespace = string(*spec.TargetRef.Namespace) + } + return fmt.Sprintf("%s-%s-%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name), targetNamespace, string(spec.TargetRef.Name)) +} + func (r *rateLimitPolicyCache) updateXdsCache(label string) bool { rlsConf := r.generateRateLimitConfig() version := fmt.Sprint(rand.Int(rand.Reader, maxRandomBigInt())) diff --git a/common-controller/internal/xds/server.go b/common-controller/internal/xds/server.go index 24be21d0ad..457f754d42 100644 --- a/common-controller/internal/xds/server.go +++ b/common-controller/internal/xds/server.go @@ -33,8 +33,10 @@ import ( wso2_cache "github.com/wso2/apk/adapter/pkg/discovery/protocol/cache/v3" eventhubTypes "github.com/wso2/apk/adapter/pkg/eventhub/types" + "github.com/wso2/apk/common-controller/internal/loggers" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + apimachiner_types "k8s.io/apimachinery/pkg/types" ) // EnvoyInternalAPI struct use to hold envoy resources and adapter internal resources @@ -163,6 +165,15 @@ func UpdateRateLimitXDSCacheForCustomPolicies(customRateLimitPolicies dpv1alpha1 } } +// UpdateRateLimitXDSCacheForAIRatelimitPolicies updates the xDS cache of the RateLimiter for AI ratelimit policies. +func UpdateRateLimitXDSCacheForAIRatelimitPolicies(aiRatelimitPolicySpecs map[apimachiner_types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + loggers.LoggerAPKOperator.Infof("000000") + if len(aiRatelimitPolicySpecs) != 0 { + loggers.LoggerAPKOperator.Infof("1111111") + rlsPolicyCache.ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRatelimitPolicySpecs) + } +} + // DeleteAPILevelRateLimitPolicies delete the ratelimit xds cache func DeleteAPILevelRateLimitPolicies(resolveRatelimitPolicyList []dpv1alpha1.ResolveRateLimitAPIPolicy) { diff --git a/common-go-libs/PROJECT b/common-go-libs/PROJECT index 51d234c0e9..5aee973d41 100644 --- a/common-go-libs/PROJECT +++ b/common-go-libs/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: wso2.com layout: - go.kubebuilder.io/v3 @@ -176,4 +180,13 @@ resources: kind: Subscription path: github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3 version: v1alpha3 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: wso2.com + group: dp + kind: AIRateLimitPolicy + path: github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3 + version: v1alpha3 version: "3" diff --git a/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go b/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go new file mode 100644 index 0000000000..200c4ee9bc --- /dev/null +++ b/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy +type AIRateLimitPolicySpec struct { + Override *AIRateLimit `json:"override,omitempty"` + Default *AIRateLimit `json:"default,omitempty"` + TargetRef gwapiv1b1.PolicyTargetReference `json:"targetRef,omitempty"` +} + +// AIRateLimit defines the AI ratelimit configuration +type AIRateLimit struct { + Organization string `json:"organization,omitempty"` + TokenCount *TokenCount `json:"tokenCount,omitempty"` + RequestCount *RequestCount `json:"requestCount,omitempty"` +} + +// TokenCount defines the Token based ratelimit configuration +type TokenCount struct { + // Unit is the unit of the requestsPerUnit + // + // +kubebuilder:validation:Enum=Minute;Hour;Day + Unit string `json:"unit,omitempty"` + + // RequestTokenCount specifies the maximum number of tokens allowed + // in AI requests within a given unit of time. This value limits the + // token count sent by the client to the AI service over the defined period. + // + // +kubebuilder:validation:Minimum=1 + RequestTokenCount uint32 `json:"requestTokenCount,omitempty"` + + // ResponseTokenCount specifies the maximum number of tokens allowed + // in AI responses within a given unit of time. This value limits the + // token count received by the client from the AI service over the defined period. + // + // +kubebuilder:validation:Minimum=1 + ResponseTokenCount uint32 `json:"responseTokenCount,omitempty"` + + // TotalTokenCount represents the maximum allowable total token count + // for both AI requests and responses within a specified unit of time. + // This value sets the limit for the number of tokens exchanged between + // the client and AI service during the defined period. + // + // +kubebuilder:validation:Minimum=1 + TotalTokenCount uint32 `json:"totalTokenCount,omitempty"` +} + +// AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy +type AIRateLimitPolicyStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// AIRateLimitPolicy is the Schema for the airatelimitpolicies API +type AIRateLimitPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AIRateLimitPolicySpec `json:"spec,omitempty"` + Status AIRateLimitPolicyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// AIRateLimitPolicyList contains a list of AIRateLimitPolicy +type AIRateLimitPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AIRateLimitPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AIRateLimitPolicy{}, &AIRateLimitPolicyList{}) +} \ No newline at end of file diff --git a/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go b/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go index e0fda22ead..5b95f57d6a 100644 --- a/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go +++ b/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go @@ -132,6 +132,131 @@ func (in *AIProviderStatus) DeepCopy() *AIProviderStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimit) DeepCopyInto(out *AIRateLimit) { + *out = *in + if in.TokenCount != nil { + in, out := &in.TokenCount, &out.TokenCount + *out = new(TokenCount) + **out = **in + } + if in.RequestCount != nil { + in, out := &in.RequestCount, &out.RequestCount + *out = new(RequestCount) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimit. +func (in *AIRateLimit) DeepCopy() *AIRateLimit { + if in == nil { + return nil + } + out := new(AIRateLimit) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicy) DeepCopyInto(out *AIRateLimitPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicy. +func (in *AIRateLimitPolicy) DeepCopy() *AIRateLimitPolicy { + if in == nil { + return nil + } + out := new(AIRateLimitPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AIRateLimitPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicyList) DeepCopyInto(out *AIRateLimitPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AIRateLimitPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicyList. +func (in *AIRateLimitPolicyList) DeepCopy() *AIRateLimitPolicyList { + if in == nil { + return nil + } + out := new(AIRateLimitPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AIRateLimitPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicySpec) DeepCopyInto(out *AIRateLimitPolicySpec) { + *out = *in + if in.Override != nil { + in, out := &in.Override, &out.Override + *out = new(AIRateLimit) + (*in).DeepCopyInto(*out) + } + if in.Default != nil { + in, out := &in.Default, &out.Default + *out = new(AIRateLimit) + (*in).DeepCopyInto(*out) + } + in.TargetRef.DeepCopyInto(&out.TargetRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicySpec. +func (in *AIRateLimitPolicySpec) DeepCopy() *AIRateLimitPolicySpec { + if in == nil { + return nil + } + out := new(AIRateLimitPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicyStatus) DeepCopyInto(out *AIRateLimitPolicyStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicyStatus. +func (in *AIRateLimitPolicyStatus) DeepCopy() *AIRateLimitPolicyStatus { + if in == nil { + return nil + } + out := new(AIRateLimitPolicyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *APIPolicy) DeepCopyInto(out *APIPolicy) { *out = *in @@ -622,6 +747,21 @@ func (in *SubscriptionRateLimitPolicy) DeepCopy() *SubscriptionRateLimitPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TokenCount) DeepCopyInto(out *TokenCount) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenCount. +func (in *TokenCount) DeepCopy() *TokenCount { + if in == nil { + return nil + } + out := new(TokenCount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ValueDetails) DeepCopyInto(out *ValueDetails) { *out = *in diff --git a/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml b/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b30f687699 --- /dev/null +++ b/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml @@ -0,0 +1,185 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: airatelimitpolicies.dp.wso2.com +spec: + group: dp.wso2.com + names: + kind: AIRateLimitPolicy + listKind: AIRateLimitPolicyList + plural: airatelimitpolicies + singular: airatelimitpolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AIRateLimitPolicy is the Schema for the airatelimitpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy + properties: + default: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the rule for request count quota. + properties: + requestsPerUnit: + format: int32 + type: integer + unit: + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + override: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the rule for request count quota. + properties: + requestsPerUnit: + format: int32 + type: integer + unit: + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + targetRef: + description: PolicyTargetReference identifies an API object to apply + a direct or inherited policy to. This should be used as part of + Policy resources that can target Gateway API resources. For more + information on how this policy attachment model works, and a sample + Policy resource, refer to the policy attachment documentation for + Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace is the namespace of the referent. When + unspecified, the local namespace is inferred. Even when policy + targets a resource in a different namespace, it MUST only apply + to traffic originating from the same namespace as the policy. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + type: object + status: + description: AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/common-go-libs/config/crd/kustomization.yaml b/common-go-libs/config/crd/kustomization.yaml index 10ba50c52e..5c3c767fc1 100644 --- a/common-go-libs/config/crd/kustomization.yaml +++ b/common-go-libs/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/dp.wso2.com_aiproviders.yaml - bases/cp.wso2.com_subscriptions.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,12 +12,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_aiproviders.yaml #- patches/webhook_in_subscriptions.yaml +#- patches/webhook_in_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_aiproviders.yaml #- patches/cainjection_in_subscriptions.yaml +#- patches/cainjection_in_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml b/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..2d28892f72 --- /dev/null +++ b/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: airatelimitpolicies.dp.wso2.com diff --git a/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml b/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b2cde3fe7d --- /dev/null +++ b/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: airatelimitpolicies.dp.wso2.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml b/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml new file mode 100644 index 0000000000..c6afbc67c6 --- /dev/null +++ b/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-editor-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml b/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml new file mode 100644 index 0000000000..5fe04d1924 --- /dev/null +++ b/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-viewer-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - get + - list + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/common-go-libs/config/rbac/role.yaml b/common-go-libs/config/rbac/role.yaml new file mode 100644 index 0000000000..2d74cacc5b --- /dev/null +++ b/common-go-libs/config/rbac/role.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/finalizers + verbs: + - update +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get + - patch + - update diff --git a/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml b/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml new file mode 100644 index 0000000000..4e1fcfd751 --- /dev/null +++ b/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml @@ -0,0 +1,12 @@ +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + labels: + app.kubernetes.io/name: airatelimitpolicy + app.kubernetes.io/instance: airatelimitpolicy-sample + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: operator + name: airatelimitpolicy-sample +spec: + # TODO(user): Add fields here diff --git a/common-go-libs/constants/constant.go b/common-go-libs/constants/constant.go index 14f01c9413..5450b8c852 100644 --- a/common-go-libs/constants/constant.go +++ b/common-go-libs/constants/constant.go @@ -20,6 +20,7 @@ package constants // Controller related constants const ( RatelimitController string = "RatelimitController" + AIRatelimitController string = "AIRatelimitController" ApplicationController string = "ApplicationController" SubscriptionController string = "SubscriptionController" ApplicationMappingController string = "ApplicationMappingController" diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java index c10358f336..238f744918 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java @@ -308,6 +308,8 @@ private void loadTrustStore() { private void loadTrustedCertsToTrustStore() throws IOException { String truststoreFilePath = getEnvVarConfig().getTrustedAdapterCertsPath(); + String certificatePath = "/home/wso2/security/truststore/ratelimiter.crt"; + TLSUtils.addCertsToTruststore(trustStore, certificatePath); TLSUtils.addCertsToTruststore(trustStore, truststoreFilePath); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java index a3b31bce7a..43dcade2a4 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java @@ -34,6 +34,8 @@ public class EnvVarConfig { private static final String OPA_CLIENT_PRIVATE_KEY_PATH = "OPA_CLIENT_PRIVATE_KEY_PATH"; private static final String OPA_CLIENT_PUBLIC_CERT_PATH = "OPA_CLIENT_PUBLIC_CERT_PATH"; private static final String ADAPTER_HOST = "ADAPTER_HOST"; + private static final String RATELIMITER_HOST = "RATELIMITER_HOST"; + private static final String RATELIMITER_PORT = "RATELIMITER_PORT"; private static final String ADAPTER_XDS_PORT = "ADAPTER_XDS_PORT"; private static final String COMMON_CONTROLLER_HOST = "COMMON_CONTROLLER_HOST"; private static final String COMMON_CONTROLLER_XDS_PORT = "COMMON_CONTROLLER_XDS_PORT"; @@ -68,6 +70,8 @@ public class EnvVarConfig { private static final String DEFAULT_ENFORCER_PUBLIC_CERT_PATH = "/home/wso2/security/keystore/mg.pem"; private static final String DEFAULT_ENFORCER_REGION_ID = "UNKNOWN"; private static final String DEFAULT_ADAPTER_HOST = "adapter"; + private static final String DEFAULT_RATELIMITER_HOST = "apk-test-wso2-apk-ratelimiter-service.apk.svc"; + private static final String DEFAULT_RATELIMITER_PORT = "8091"; private static final String DEFAULT_ADAPTER_XDS_PORT = "18000"; private static final String DEFAULT_COMMON_CONTROLLER_HOST = "common-controller"; private static final String DEFAULT_COMMON_CONTROLLER_XDS_PORT = "18002"; @@ -100,6 +104,8 @@ public class EnvVarConfig { private final String opaClientPrivateKeyPath; private final String opaClientPublicKeyPath; private final String adapterHost; + private final String ratelimiterHost; + private final String ratelimiterPort; private final String commonControllerHost; private final String enforcerLabel; private final String adapterXdsPort; @@ -144,6 +150,8 @@ private EnvVarConfig() { DEFAULT_ENFORCER_PUBLIC_CERT_PATH); enforcerLabel = retrieveEnvVarOrDefault(ENFORCER_LABEL, DEFAULT_ENFORCER_LABEL); adapterHost = retrieveEnvVarOrDefault(ADAPTER_HOST, DEFAULT_ADAPTER_HOST); + ratelimiterHost = retrieveEnvVarOrDefault(RATELIMITER_HOST, DEFAULT_RATELIMITER_HOST); + ratelimiterPort = retrieveEnvVarOrDefault(RATELIMITER_PORT, DEFAULT_RATELIMITER_PORT); adapterHostname = retrieveEnvVarOrDefault(ADAPTER_HOST_NAME, DEFAULT_ADAPTER_HOST_NAME); adapterXdsPort = retrieveEnvVarOrDefault(ADAPTER_XDS_PORT, DEFAULT_ADAPTER_XDS_PORT); commonControllerHost = retrieveEnvVarOrDefault(COMMON_CONTROLLER_HOST, DEFAULT_COMMON_CONTROLLER_HOST); @@ -240,6 +248,19 @@ public String getAdapterHost() { return adapterHost; } + public String getRatelimiterHost() { + return ratelimiterHost; + } + + public int getRatelimiterPort() { + try { + int port = Integer.parseInt(ratelimiterPort); + return port; + } catch (NumberFormatException e) { + return Integer.parseInt(DEFAULT_RATELIMITER_PORT); + } + } + public String getCommonControllerHost() { return commonControllerHost; } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java new file mode 100644 index 0000000000..bda06b8e84 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2020, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.apk.enforcer.grpc; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.Value; +import io.envoyproxy.envoy.service.ext_proc.v3.BodyMutation; +import io.envoyproxy.envoy.service.ext_proc.v3.BodyResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.CommonResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.ExternalProcessorGrpc; +import io.envoyproxy.envoy.service.ext_proc.v3.HeadersResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingRequest; +import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingResponse; +import io.grpc.stub.StreamObserver; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.wso2.apk.enforcer.grpc.client.RatelimitClient; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This is the gRPC server written to match with the envoy ext-authz filter proto file. Envoy proxy call this service. + * This is the entry point to the filter chain process for a request. + */ +public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProcessorImplBase { + private static final Logger logger = LogManager.getLogger(ExternalProcessorService.class); + private static final String DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT = "airequesttokencount"; + private static final String DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT = "airesponsetokencount"; + private static final String DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT = "aitotaltokencount"; + RatelimitClient ratelimitClient = new RatelimitClient(); + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + FilterMetadata filterMetadata = new FilterMetadata(); + System.out.println("process ...."); + return new StreamObserver() { + + @Override + public void onNext(ProcessingRequest request) { + System.out.println("on next ...."); + ProcessingRequest.RequestCase r = request.getRequestCase(); + System.out.println("case: " + r.name()); + switch (r) { + case REQUEST_HEADERS: + if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ + Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); + FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); + System.out.println("Metadata generated: "+ metadata); + filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; + filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; + filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; + } + responseObserver.onNext(ProcessingResponse.newBuilder().build()); + case RESPONSE_BODY: + if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ + Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); + FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); + System.out.println("Metadata generated: "+ metadata); + filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; + filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; + filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; + } + System.out.println("In the response flow metadata descirtor:" + filterMetadata.backendBasedAIRatelimitDescriptorValue); + if (request.hasResponseBody()) { + String body = request.getResponseBody().getBody().toStringUtf8(); +// System.out.println("Body: " + body); + Usage usage = extractUsageFromBody(body, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); + if (usage == null) { + logger.error("Usage details not found.."); + System.out.println("Usage details not found.."); + responseObserver.onCompleted(); + return; + } + System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); + if (filterMetadata.enableBackendBasedAIRatelimit) { + List configs = new ArrayList<>(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + ratelimitClient.shouldRatelimit(configs); + } + responseObserver.onCompleted(); + } else { + System.out.println("Request does not have response body"); + responseObserver.onCompleted(); + } + + } + } + + @Override + public void onError(Throwable err) { + System.out.println("on error ...."+ err.getLocalizedMessage() + " " + err.getMessage() + " " + err.toString()+ " ****"); + } + + @Override + public void onCompleted() { + System.out.println("on completed ...."); + responseObserver.onCompleted(); + } + }; + } + + protected BodyResponse prepareBodyResponse() { + return BodyResponse.newBuilder() + .setResponse( + CommonResponse.newBuilder() + .setStatus(CommonResponse.ResponseStatus.CONTINUE) + .setBodyMutation(BodyMutation.newBuilder().build()) + .build()) + .build(); + } + + protected HeadersResponse prepareHeadersResponse() { + return HeadersResponse.newBuilder() + .setResponse( + CommonResponse.newBuilder() + .setStatus(CommonResponse.ResponseStatus.CONTINUE) + .setBodyMutation(BodyMutation.newBuilder().build()) + .build()) + .build(); + } + + // The FilterMetadata class as per your request + private static class FilterMetadata { + boolean enableSubscriptionBasedAIRatelimit; + boolean enableBackendBasedAIRatelimit; + String backendBasedAIRatelimitDescriptorValue; + @Override + public String toString() { + return "FilterMetadata{" + + "enableSubscriptionBasedAIRatelimit=" + enableSubscriptionBasedAIRatelimit + + ", enableBackendBasedAIRatelimit=" + enableBackendBasedAIRatelimit + + ", backendBasedAIRatelimitDescriptorValue='" + backendBasedAIRatelimitDescriptorValue + '\'' + + '}'; + } + } + + // Method to parse the string and create FilterMetadata object + public static FilterMetadata convertStringToFilterMetadata(String input) { + FilterMetadata metadata = new FilterMetadata(); + + // Regex patterns to extract specific fields + String backendValuePattern = "key: \"BackendBasedAIRatelimitDescriptorValue\".*?string_value: \"(.*?)\""; + String enableBackendPattern = "key: \"EnableBackendBasedAIRatelimit\".*?string_value: \"(.*?)\""; + String enableSubscriptionPattern = "key: \"EnableSubscriptionBasedAIRatelimit\".*?string_value: \"(.*?)\""; + + // Extract and assign to the FilterMetadata object + metadata.backendBasedAIRatelimitDescriptorValue = extractValue(input, backendValuePattern); + metadata.enableBackendBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableBackendPattern)); + metadata.enableSubscriptionBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableSubscriptionPattern)); + + return metadata; + } + + // Helper method to extract value based on a regex pattern + private static String extractValue(String input, String pattern) { + Pattern p = Pattern.compile(pattern); + Matcher m = p.matcher(input); + if (m.find()) { + return m.group(1); + } + return null; + } + + + public static void main(String[] args) { + String input = "filter_metadata { key: \"envoy.filters.http.ext_proc\" value { fields { key: \"BackendBasedAIRatelimitDescriptorValue\" value { string_value: \"default-apk-backend-ratelimit-xxx-apk-backend-93fa16a00dbec9ff438aa40e6358d91dcdc22f48-api\" } } fields { key: \"EnableBackendBasedAIRatelimit\" value { string_value: \"true\" } } fields { key: \"EnableSubscriptionBasedAIRatelimit\" value { string_value: \"false\" } } } }"; + input = "{ \"choices\":[ { \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } }, \"finish_reason\":\"stop\", \"index\":0, \"logprobs\":null, \"message\":{ \"content\":\"Arr, matey! Ye be askin' a great question. Care for a parrot be a task that requires careful attention and love. Here be some tips to keep yer feathered friend happy and healthy: 1. Provide a proper cage: A parrot needs an adequate-sized cage to freely stretch its wings. Make sure it has enough space for perching, playing, and spreading those beautiful feathers. Also, ensure the bars are close enough together to prevent escape. 2. Nourishing grub: A parrot's diet be important. Offer a balanced diet of high-quality parrot pellets, fresh fruits, vegetables, and some seeds. Avoid avacados, chocolate, caffeine, and anythin' toxic to a bird's delicate system. 3. Fresh water: Change the water in yer parrot's bowl daily, matey. Keeps it clean and fresh. Parrots love to dunk their beaks, so ensure they have ample water for sippin' and splish-splashin'. 4. Feathered entertainment: Parrots be social creatures and need mental stimulation. Provide 'em with plenty of toys, such as ropes, bells, and puzzle toys, to keep 'em entertained. Rotate the toys frequently to avoid boredom. 5. Avast, matey! Give 'em attention: Parrots be fond of interaction with their human companions. Spend time talkin' to 'em, singin' shanties, and makin' 'em feel loved. They may even learn a few words or phrases! 6. Exercise be important: Encourage yer parrot to exercise its wings, me hearty. Free-flyin' in a safe, enclosed area be ideal, or let 'em out of their cage for supervised playtime. 7. Regular vet visits: Aye, take yer parrot to a qualified avian vet for regular check-ups. They'll make sure yer feathered friend's health be in shipshape and suggest any necessary vaccinations or treatments. 8. Aye, watch for signs of illness: Keep a keen eye on yer parrot for any signs of illness, such as changes in appetite, behavior, or feather condition. If ye spot any concerns, consult an avian vet right quick. Remember, matey, each parrot be unique, so get to know yer bird and pay attention to its specific needs. Aye, with proper care, ye and yer parrot will forge a bond that be stronger than the mightiest of pirate ships. Fair winds and happy parrot keepin'!\", \"role\":\"assistant\" } } ], \"created\":1724232516, \"id\":\"chatcmpl-9ybvo8dQte9Hb0IkD2NCaOME9Q1LH\", \"model\":\"gpt-35-turbo\", \"object\":\"chat.completion\", \"prompt_filter_results\":[ { \"prompt_index\":0, \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } } } ], \"system_fingerprint\":null, \"usage\":{ \"completion_tokens\":514, \"prompt_tokens\":33, \"total_tokens\":547 } }"; + Usage usage = extractUsageFromBody(input, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); + System.out.println(usage.completion_tokens); +// FilterMetadata metadata = convertStringToFilterMetadata(input); +// System.out.println(metadata.backendBasedAIRatelimitDescriptorValue); // Printing the FilterMetadata object + } + + public static String sanitize(String input) { + // Replace all newline characters and tabs with a space + return input.replaceAll("[\\t\\n\\r]+", " ").trim(); + } + + private static Usage extractUsageFromBody(String body, String completionTokenPath, String promptTokenPath, String totalTokenPath) { + body = sanitize(body); + ObjectMapper mapper = new ObjectMapper(); + try { + Usage usage = new Usage(); + // Parse the JSON string + JsonNode rootNode = mapper.readTree(body); + // Extract prompt token count + String[] keysForPromtTokens = promptTokenPath.split("\\."); + JsonNode currentNodeForPromtToken = null; + if (rootNode.has(keysForPromtTokens[0])) { + currentNodeForPromtToken = rootNode.get(keysForPromtTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForPromtTokens.length; i++) { + if (currentNodeForPromtToken.has(keysForPromtTokens[i])) { + currentNodeForPromtToken = currentNodeForPromtToken.get(keysForPromtTokens[i]); + } else { + return null; + } + } + usage.setPrompt_tokens(currentNodeForPromtToken.asInt()); + + // Extract completion token count + String[] keysForCompletionTokens = completionTokenPath.split("\\."); + JsonNode currentNodeForCompletionToken = null; + if (rootNode.has(keysForCompletionTokens[0])) { + currentNodeForCompletionToken = rootNode.get(keysForCompletionTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForCompletionTokens.length; i++) { + if (currentNodeForCompletionToken.has(keysForCompletionTokens[i])) { + currentNodeForCompletionToken = currentNodeForCompletionToken.get(keysForCompletionTokens[i]); + } else { + return null; + } + } + usage.setCompletion_tokens(currentNodeForCompletionToken.asInt()); + + // Extract total token count + String[] keysForTotalTokens = totalTokenPath.split("\\."); + JsonNode currentNodeForTotalToken = null; + if (rootNode.has(keysForTotalTokens[0])) { + currentNodeForTotalToken = rootNode.get(keysForTotalTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForTotalTokens.length; i++) { + if (currentNodeForTotalToken.has(keysForTotalTokens[i])) { + currentNodeForTotalToken = currentNodeForTotalToken.get(keysForTotalTokens[i]); + } else { + return null; + } + } + usage.setTotal_tokens(currentNodeForTotalToken.asInt()); + System.out.println("Usage extracted: "+ usage); + return usage; + + } catch (JsonProcessingException e) { + logger.error(String.format("Unexpected error while extracting usage from the body: %s", body), e); + return null; + } + } + + public static class Usage { + private int completion_tokens; + private int prompt_tokens; + private int total_tokens; + + // Getters and Setters + public int getCompletion_tokens() { + return completion_tokens; + } + + public void setCompletion_tokens(int completion_tokens) { + this.completion_tokens = completion_tokens; + } + + public int getPrompt_tokens() { + return prompt_tokens; + } + + public void setPrompt_tokens(int prompt_tokens) { + this.prompt_tokens = prompt_tokens; + } + + public int getTotal_tokens() { + return total_tokens; + } + + public void setTotal_tokens(int total_tokens) { + this.total_tokens = total_tokens; + } + + @Override + public String toString() { + return String.format("%s_%s_%s", prompt_tokens, completion_tokens, total_tokens); + } + } + +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java new file mode 100644 index 0000000000..3034296060 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -0,0 +1,97 @@ +package org.wso2.apk.enforcer.grpc.client; + +import io.envoyproxy.envoy.extensions.common.ratelimit.v3.RateLimitDescriptor; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitRequest; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitServiceGrpc; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitResponse; +import io.grpc.ManagedChannel; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import org.wso2.apk.enforcer.config.ConfigHolder; + +import java.io.File; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.net.ssl.SSLException; + +public class RatelimitClient { + RateLimitServiceGrpc.RateLimitServiceBlockingStub stub; + private final ExecutorService executorService; // Add an ExecutorService field + + public RatelimitClient(){//String server, int port) { + System.out.println("Ratelimitclient construct"); + File certFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPublicKeyPath()).toFile(); + File keyFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPrivateKeyPath()).toFile(); + SslContext sslContext = null; + try { + sslContext = GrpcSslContexts + .forClient() + .trustManager(ConfigHolder.getInstance().getTrustManagerFactory()) + .keyManager(certFile, keyFile) + .build(); + } catch (SSLException e) { + System.out.println("Error while generating SSL Context."+ e); + } + String rlHost = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterHost(); + int port = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterPort(); + System.out.println("rl host and port "+ rlHost + port); + ManagedChannel channel = NettyChannelBuilder.forAddress(rlHost, port) + .useTransportSecurity() + .sslContext(sslContext) + .build(); + this.stub = RateLimitServiceGrpc.newBlockingStub(channel); + // Initialize the ExecutorService + this.executorService = Executors.newFixedThreadPool(10); + } + + public void shouldRatelimit(List configs) { +// System.out.println("RL task submitted"); + executorService.submit(() -> { + System.out.println("Ratelimitclient test"); + for (KeyValueHitsAddend config : configs) { + System.out.println("For: " + config.getKey()); + RateLimitDescriptor descriptor = RateLimitDescriptor.newBuilder() + .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()) + .build(); + RateLimitRequest rateLimitRequest = RateLimitRequest.newBuilder() + .addDescriptors(descriptor) + .setDomain("Default") + .setHitsAddend(config.getHitsAddend()) + .build(); + RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); + System.out.println(rateLimitResponse.getOverallCode()); + } + }); + System.out.println("RL task submitted"); + + } + + public static class KeyValueHitsAddend { + private String key; + private String value; + private int hitsAddend; + + public KeyValueHitsAddend(String key, String value, int hitsAddend) { + this.key = key; + this.value = value; + this.hitsAddend = hitsAddend; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + public int getHitsAddend() { + return hitsAddend; + } + } + + +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java index 6f31ffc2de..d849dc648a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java @@ -36,6 +36,7 @@ import org.wso2.apk.enforcer.config.dto.ThreadPoolConfig; import org.wso2.apk.enforcer.discovery.ConfigDiscoveryClient; import org.wso2.apk.enforcer.grpc.ExtAuthService; + import org.wso2.apk.enforcer.grpc.ExternalProcessorService; import org.wso2.apk.enforcer.grpc.HealthService; import org.wso2.apk.enforcer.grpc.interceptors.AccessLogInterceptor; import org.wso2.apk.enforcer.grpc.interceptors.OpenTelemetryInterceptor; @@ -158,12 +159,14 @@ private static Server initServer() throws SSLException { EnforcerWorkerPool enforcerWorkerPool = new EnforcerWorkerPool(threadPoolConfig.getCoreSize(), threadPoolConfig.getMaxSize(), threadPoolConfig.getKeepAliveTime(), threadPoolConfig.getQueueSize(), Constants.EXTERNAL_AUTHZ_THREAD_GROUP, Constants.EXTERNAL_AUTHZ_THREAD_ID); + System.out.println("test"); return NettyServerBuilder.forPort(authServerConfig.getPort()) .keepAliveTime(authServerConfig.getKeepAliveTime(), TimeUnit.SECONDS).bossEventLoopGroup(bossGroup) .workerEventLoopGroup(workerGroup) .addService(ServerInterceptors.intercept(new ExtAuthService(), new OpenTelemetryInterceptor(), new AccessLogInterceptor())) .addService(new HealthService()) + .addService(new ExternalProcessorService()) // .addService(ServerInterceptors.intercept(new WebSocketFrameService(), new AccessLogInterceptor())) .maxInboundMessageSize(authServerConfig.getMaxMessageSize()) .maxInboundMetadataSize(authServerConfig.getMaxHeaderLimit()).channelType(NioServerSocketChannel.class) diff --git a/gateway/router/Dockerfile b/gateway/router/Dockerfile index 325a988056..5d45304b53 100644 --- a/gateway/router/Dockerfile +++ b/gateway/router/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -FROM envoyproxy/envoy:v1.29.1 +FROM envoyproxy/envoy:v1.31.0 LABEL maintainer="WSO2 Docker Maintainers " RUN apt-get update && apt-get upgrade -y && apt-get install -y curl diff --git a/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml b/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml new file mode 100644 index 0000000000..8eb400bcdc --- /dev/null +++ b/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml @@ -0,0 +1,201 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: airatelimitpolicies.dp.wso2.com +spec: + group: dp.wso2.com + names: + kind: AIRateLimitPolicy + listKind: AIRateLimitPolicyList + plural: airatelimitpolicies + singular: airatelimitpolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AIRateLimitPolicy is the Schema for the airatelimitpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy + properties: + default: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the request based ratelimit + configuration + properties: + requestsPerUnit: + description: RequestPerUnit is the number of requests allowed + per unit time + format: int32 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + override: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the request based ratelimit + configuration + properties: + requestsPerUnit: + description: RequestPerUnit is the number of requests allowed + per unit time + format: int32 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + targetRef: + description: PolicyTargetReference identifies an API object to apply + a direct or inherited policy to. This should be used as part of + Policy resources that can target Gateway API resources. For more + information on how this policy attachment model works, and a sample + Policy resource, refer to the policy attachment documentation for + Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace is the namespace of the referent. When + unspecified, the local namespace is inferred. Even when policy + targets a resource in a different namespace, it MUST only apply + to traffic originating from the same namespace as the policy. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + type: object + status: + description: AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 3d6fc60622..bf6217a4ff 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -70,6 +70,8 @@ spec: value: {{ template "apk-helm.resource.prefix" . }}-common-controller-service.{{ .Release.Namespace }}.svc - name: COMMON_CONTROLLER_HOST value: {{ template "apk-helm.resource.prefix" . }}-common-controller-service.{{ .Release.Namespace }}.svc + - name: RATELIMITER_HOST + value: {{ template "apk-helm.resource.prefix" . }}-ratelimiter-service.{{ .Release.Namespace }}.svc - name: ENFORCER_PRIVATE_KEY_PATH value: /home/wso2/security/keystore/enforcer.key - name: ENFORCER_PUBLIC_CERT_PATH @@ -80,6 +82,8 @@ spec: value: "/home/wso2/security/truststore" - name: ADAPTER_XDS_PORT value : "18000" + - name: RATELIMITER_PORT + value : "8091" - name: COMMON_CONTROLLER_XDS_PORT value : "18002" - name: COMMON_CONTROLLER_REST_PORT @@ -96,9 +100,9 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml + value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers }} {{- $defaultPublisherSecretName := "" }} @@ -255,6 +259,22 @@ spec: mountPath: /home/wso2/security/truststore/idp-tls.pem subPath: {{ .Values.wso2.apk.idp.tls.fileName }} {{ end }} + {{ if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.dp.ratelimiter.enabled }} + - name: ratelimiter-truststore-secret-volume + mountPath: /home/wso2/security/truststore/ratelimiter.crt + {{- if and .Values.wso2.apk.dp.ratelimiter.deployment.configs .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls }} + subPath: {{ .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls.certFilename | default "tls.crt" }} + {{- else }} + subPath: tls.crt + {{- end }} + - name: ratelimiter-truststore-secret-volume + mountPath: /home/wso2/security/truststore/ratelimiter-ca.crt + {{- if and .Values.wso2.apk.dp.ratelimiter.deployment.configs .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls }} + subPath: {{ .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls.certCAFilename | default "ca.crt" }} + {{- else }} + subPath: ca.crt + {{- end }} + {{ end }} readinessProbe: exec: command: [ "sh", "check_health.sh" ] diff --git a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml index 751434027b..2581e93039 100644 --- a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml +++ b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml @@ -92,6 +92,15 @@ rules: - apiGroups: [ "dp.wso2.com" ] resources: [ "ratelimitpolicies/status" ] verbs: [ "get","patch","update" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies" ] + verbs: [ "get","list","watch","update","delete","create" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies/finalizers" ] + verbs: [ "update" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies/status" ] + verbs: [ "get","patch","update" ] - apiGroups: [ "coordination.k8s.io" ] resources: [ "leases" ] verbs: [ "get","list","watch","update","patch","create","delete" ] diff --git a/libs.versions.toml b/libs.versions.toml index 100dd115ac..64f34aa1d2 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -131,12 +131,12 @@ commons-logging = "1.1.1" commons-pool = "1.5.6.wso2v1" commons-validator = "1.7" cxf = "3.5.4" -envoyproxy = "1.0.42" +envoyproxy = "1.0.46" fasterxml-woodstox="6.4.0" everit = "1.5.0.wso2.v2" geronimo = "1.1.1.wso2v1" graphql = "21.1" -grpc = "1.53.0" +grpc = "1.62.2" gson = "2.10" guava = "32.1.2-jre" hibernate-validator = "5.4.3.Final" diff --git a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature deleted file mode 100644 index b96dff9607..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature +++ /dev/null @@ -1,84 +0,0 @@ -Feature: API Definition Endpoint - Scenario: Testing default API definition endpoint - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/api_definition_default.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" - And I eventually receive 200 response code, not accepting - |429| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/api-definition" with body "" - And I eventually receive 200 response code, not accepting - |429| - - Scenario: Testing custom API definition endpoint - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/api_definition_custom.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/3.14/docs" with body "" - And I eventually receive 200 response code, not accepting - |429| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/docs" with body "" - And I eventually receive 200 response code, not accepting - |429| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/api-definition" with body "" - And I eventually receive 404 response code, not accepting - |429| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/3.14/api-definition" with body "" - And I eventually receive 404 response code, not accepting - |429| - - Scenario: Testing a deleted production endpoint - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/api_definition_default_without_production.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I wait for 1 minute - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" - Then the response status code should be 404 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/api-definition" with body "" - Then the response status code should be 404 - - Scenario: Testing a deleted production endpoint - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/api_definition_default.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - When I use the APK Conf file "artifacts/apk-confs/api_definition_default_without_sandbox.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I wait for 1 minute - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" - Then the response status code should be 404 - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-definition-default/api-definition" with body "" - Then the response status code should be 404 - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | custom-api-definition-endpoint-test | 202 | - | default-api-definition-endpoint-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature deleted file mode 100644 index 36304a6ab4..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature +++ /dev/null @@ -1,72 +0,0 @@ -Feature: API Subscription Feature - Scenario: testing api subscriptions. - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/subscription-api.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 403 response code, not accepting - |429| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - Then the response status code should be 403 - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - Then the response status code should be 403 - - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120006" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120006-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 403 response code, not accepting - |200| - |429| - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120007" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120007-token}| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 403 response code, not accepting - |200| - |429| - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120008" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120008-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 403 response code, not accepting - |200| - |401| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - And I eventually receive 403 response code, not accepting - |200| - |429| - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120009" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120009-token}| - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - Then the response status code should be 403 - And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" - Then the response status code should be 403 - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | subscription-api | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature b/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature deleted file mode 100644 index a9451fa629..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature +++ /dev/null @@ -1,35 +0,0 @@ -Feature: BackendRetry - Scenario: Testing backend retry - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/backend_retry_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" - And I eventually receive 200 response code, not accepting - |429| - And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/set-retry-count" with body "{\"count\": 4}" - Then the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/reset" with body "" - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" - Then the response status code should be 500 - And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/set-retry-count" with body "{\"count\": 2}" - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" - Then the response status code should be 200 - - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | backend-retry-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature b/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature deleted file mode 100644 index 1c3279a9a0..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature +++ /dev/null @@ -1,48 +0,0 @@ -Feature: BackendTimeout - Scenario: Testing backend timeout - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/backend_timeout_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/2" with body "" - And I eventually receive 504 response code, not accepting - |429| - |200| - And the response body should contain "timeout" - And I send "GET" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/0" with body "" - And I eventually receive 200 response code, not accepting - |429| - |500| - And I send "POST" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/3" with body "" - And I eventually receive 504 response code, not accepting - |429| - |200| - And the response body should contain "timeout" - And I send "POST" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/1" with body "" - And I eventually receive 200 response code, not accepting - |429| - |500| - And I send "PUT" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/4" with body "" - And I eventually receive 504 response code, not accepting - |429| - |200| - And the response body should contain "timeout" - And I send "PUT" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/2" with body "" - And I eventually receive 200 response code, not accepting - |429| - |500| - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | backend-timeout-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature b/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature deleted file mode 100644 index 603680b21f..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature +++ /dev/null @@ -1,43 +0,0 @@ -Feature: Basic auth - Scenario: Testing API level and resource level basic auth header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/basic_auth_conf.yaml" - And the definition file "artifacts/definitions/basic_auth_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "\"Authorization\": \"Basic YWRtaW46YWRtaW4=\"" - And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/get" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "\"Authorization\": \"Basic ZHNmZHNmc2Rmc2RmOmFkbWlu\"" - And I send "POST" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/post" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "\"Authorization\": \"Basic YWRtaW46YWRtaW4=\"" - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | basic-auth-api-test | 202 | - - Scenario: Testing undeployed API - Given The system is ready - And I have a valid subscription - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/employee/" with body "" - And I eventually receive 404 response code, not accepting - | 200 | - diff --git a/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature b/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature deleted file mode 100644 index 8fb5f7f62d..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature +++ /dev/null @@ -1,138 +0,0 @@ -Feature: API Deployment and invocation - Scenario: Deploying an API and basic http method invocations - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" - And the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/test/" with body "" - And the response status code should be 404 - And I send "PUT" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" - And the response status code should be 200 - And I send "DELETE" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" - And the response status code should be 200 - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" - And the response status code should be 401 - And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" - And the response status code should be 401 - And I send "PUT" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" - And the response status code should be 401 - And I send "DELETE" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" - And the response status code should be 401 - - Scenario: Deploying an API with new version - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/new_version_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - When I use the APK Conf file "artifacts/apk-confs/new_version_conf2.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I wait for next minute - And I send "GET" request to "https://default.gw.wso2.com:9095/test-version/1.0/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-version/2.0/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - - Scenario: Deploying an API with default version - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/default_version_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/" with body "" - And the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/3.14/test/" with body "" - And the response status code should be 404 - And I send "PUT" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/12" with body "" - And the response status code should be 200 - And I send "DELETE" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/12" with body "" - And the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-default/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/employee/" with body "" - And the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/test/" with body "" - And the response status code should be 404 - And I send "PUT" request to "https://default.gw.wso2.com:9095/test-default/employee/12" with body "" - And the response status code should be 200 - And I send "DELETE" request to "https://default.gw.wso2.com:9095/test-default/employee/12" with body "" - And the response status code should be 200 - - Scenario: Scope Validation - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/employees_scope_test_conf.yaml" - And the definition file "artifacts/definitions/employees_scope_test_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithoutscope/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope1/" with body "" - And the response status code should be 403 - Given I have a valid subscription with scopes - | scope1 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithoutscope/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope1/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope2/" with body "" - And I eventually receive 403 response code, not accepting - | 200 | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscopes/" with body "" - And I eventually receive 403 response code, not accepting - | 429 | - Given I have a valid subscription with scopes - | scope1 | - | scope2 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscopes/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | f7996dce4ac15e2af0f8ee14546c4f72988eddae | 202 | - | default-version-api-test | 202 | - | emp-api-test-scope | 202 | - | version-api-test | 202 | - | version-api-test2 | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CORS.feature b/test/cucumber-tests/src/test/resources/tests/api/CORS.feature deleted file mode 100644 index 726fbfc236..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/CORS.feature +++ /dev/null @@ -1,60 +0,0 @@ -Feature: CORS Policy - - Scenario: Testing CORS Policy - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And I wait for next minute - And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" - And I eventually receive 204 response code, not accepting - | 429 | - And the response headers should not contain - | Access-Control-Allow-Origin | - | Access-Control-Allow-Credentials | - | Access-Control-Allow-Methods | - | Access-Control-Allow-Headers | - | Access-Control-Max-Age | - Then I set headers - | Origin | test.domain.com | - And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" - And I eventually receive 204 response code, not accepting - | 429 | - And the response headers should not contain - | Access-Control-Allow-Origin | - | Access-Control-Allow-Credentials | - | Access-Control-Allow-Methods | - | Access-Control-Allow-Headers | - | Access-Control-Max-Age | - Then I set headers - | Origin | abc.com | - And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" - And I eventually receive 204 response code, not accepting - | 429 | - And the response headers should contain - | Access-Control-Allow-Origin | abc.com | - | Access-Control-Allow-Credentials | true | - Then I set headers - | Origin | abc.com | - | Access-Control-Request-Method | GET | - And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" - And I eventually receive 200 response code, not accepting - | 429 | - And the response headers should contain - | Access-Control-Allow-Origin | abc.com | - | Access-Control-Allow-Credentials | true | - | Access-Control-Allow-Methods | GET, POST, PUT, DELETE | - | Access-Control-Allow-Headers | Content-Type, Authorization | - | Access-Control-Max-Age | 3600 | - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | cors-api-adff3dbc-2787-11ee-be56-0242ac120002 | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature deleted file mode 100644 index 927f682b08..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature +++ /dev/null @@ -1,36 +0,0 @@ -Feature: BackendRetry - Scenario: Testing backend retry - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/circuit_breaker_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/set-retry-count" with body "{\"count\": 500}" - And I eventually receive 200 response code, not accepting - |429| - And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/reset" with body "" - Then the response status code should be 200 - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" - Then the response status code should be 500 - And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/set-retry-count" with body "{\"count\": 3}" - Then the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/reset" with body "" - Then the response status code should be 200 - - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | circuit-breaker-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature deleted file mode 100644 index 39d17e6bcd..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature +++ /dev/null @@ -1,26 +0,0 @@ -Feature: circuitBreakerMaxRequest - Scenario: Testing backend timeout - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-config-dump.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I wait for 5 seconds - And I send "GET" request to "http://default.gw.wso2.com:9000/config_dump" with body "" - Then the response status code should be 200 - And the response body should contain "\"max_connections\": 1111" - And the response body should contain "\"max_pending_requests\": 1112" - And the response body should contain "\"max_requests\": 1113" - And the response body should contain "\"max_retries\": 1114" - And the response body should contain "\"max_connection_pools\": 1115" - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | circuit-breaker-config-dump | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature deleted file mode 100644 index 705430b751..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature +++ /dev/null @@ -1,47 +0,0 @@ -Feature: circuitBreakerMaxRequest - Scenario: Testing backend timeout - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-max-request-test.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/anything/test" with body "" - And I eventually receive 200 response code, not accepting - |429| - |500| - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" - Then the response status code should be 503 - - When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-max-request-test-v1.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/anything/test" with body "" - And I eventually receive 200 response code, not accepting - |429| - |500| - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" - And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" - Then the response status code should be 200 - - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | circuit-breaker-max-request-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature deleted file mode 100644 index 5b7c0ecea3..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature +++ /dev/null @@ -1,118 +0,0 @@ -Feature: Custom ratelimit - Scenario: Testing custom ratelimit - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/custom_ratelimit_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | user_id | bob | - | org_id | wso2 | - And I wait for next minute -# Request 1 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/3.14/employee" with body "" - And I eventually receive 200 response code, not accepting - | 429 | -# Request 2 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 3 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 4 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 5 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 6 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 7 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 8 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 9 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 10 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 - Then I set headers - | user_id | dummy | - | org_id | wso2 | -# Starting from Request 5 the org_id descriptor should not be counted -# Request 5 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 6 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 7 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 8 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 9 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 10 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 11 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Request 12 - for org_id descriptor - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 -# Test org_id only - And I wait for next minute strictly -# Request 1 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/3.14/employee" with body "" - Then the response status code should be 200 -# Request 2 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 3 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 4 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 5 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 6 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 7 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 8 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 9 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 10 - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 200 -# Request 11 - should be limitted - And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" - Then the response status code should be 429 - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | custom-ratelimit-api | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature b/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature deleted file mode 100644 index cc9a7c18eb..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature +++ /dev/null @@ -1,28 +0,0 @@ -Feature: API different endpoint resource level - Scenario: Testing different endpoint resource level - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/different_endpoint_resource_level.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-endpoint-resource-level/3.14/endpoint1" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/base1/endpoint1" - And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-endpoint-resource-level/endpoint2" with body "" - Then the response status code should be 200 - And the response body should contain "https://backend/anything/base2/endpoint2" - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | different-endpoint-resource-level-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature deleted file mode 100644 index 8f8c028367..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature +++ /dev/null @@ -1,34 +0,0 @@ -Feature: API different endpoint resource level - Scenario: Testing different endpoint resource level - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/different_sand_prod_endpoint.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-sand-prod-endpoint/3.14/endpoint1" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/prodr/endpoint1" - And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-sand-prod-endpoint/endpoint2" with body "" - Then the response status code should be 200 - And the response body should contain "https://backend/anything/prod/endpoint2" - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-different-sand-prod-endpoint/3.14/endpoint1" with body "" - Then the response status code should be 200 - And the response body should contain "https://backend/anything/sandr/endpoint1" - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-different-sand-prod-endpoint/endpoint2" with body "" - Then the response status code should be 200 - And the response body should contain "https://backend/anything/sand/endpoint2" - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | test-different-sand-prod-endpoint | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature deleted file mode 100644 index f9d8c30747..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature +++ /dev/null @@ -1,37 +0,0 @@ -Feature: Endpoint - Scenario: Testing API level and resource level endpoints - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/endpoint_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/endpoint/3.14/employee" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/employee" - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/endpoint/3.14/employee" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/test/employee" - And I send "POST" request to "https://default.gw.wso2.com:9095/endpoint/3.14/employee" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/test/employee" - And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/endpoint/3.14/employee" with body "" - And I eventually receive 200 response code, not accepting - |429| - And the response body should contain "https://backend/anything/employee" - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | endpoint-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature deleted file mode 100644 index 41bff9d905..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature +++ /dev/null @@ -1,25 +0,0 @@ -Feature: API Deployment with Global Interceptor - Scenario: Deploying an API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/interceptors/globalInterceptor.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "579ba27a1e03e2fdf099d1b6745e265f2d495606" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/globalinterceptor/1.0.0/get" with body "" - Then the response status code should be 200 - And the response body should contain "\"Gw-Header\": \"GW-header-value\"" - Then the response headers contains key "gw-response-header" and value "GW-response-header-value" - Scenario Outline: Undeploy an API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | 579ba27a1e03e2fdf099d1b6745e265f2d495606 | 202 | \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature b/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature deleted file mode 100644 index ea144361a5..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature +++ /dev/null @@ -1,237 +0,0 @@ -Feature: Generating APK conf for GraphQL API - Scenario: Generating APK conf using a valid GraphQL API definition - Given The system is ready - When I use the definition file "artifacts/definitions/graphql_sample_api.graphql" in resources - And generate the APK conf file for a "GRAPHQL" API - Then the response status code should be 200 - - Scenario: Deploying APK conf using a valid GraphQL API definition without a subscription resource - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_conf_without_sub.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-without-sub" - Then the response status code should be 202 - - Scenario: Deploying GraphQL API with scopes - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_scopes.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 403 response code, not accepting - | 429 | - | 500 | - Given I have a valid subscription with scopes - | wso2 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-scopes" - Then the response status code should be 202 - - Scenario: Deploying a ratelimited GraphQL API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_rl.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - Then the response status code should be 429 - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-rl" - Then the response status code should be 202 - - Scenario: Deploying multiple versions of a GraphQL API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_3.0.0.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_4.0.0.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.0.0" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/4.0.0" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | graphql-v3 | 202 | - | graphql-v4 | 202 | - - Scenario: Deploying APK conf using a valid GraphQL API definition with mTLS mandatory and valid certificate - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-mtls" - Then the response status code should be 202 - - Scenario: Deploying APK conf using a valid GraphQL API definition with mTLS mandatory and no certificate - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 401 response code, not accepting - | 200 | - | 429 | - | 500 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-mtls" - Then the response status code should be 202 - - Scenario: Deploying APK conf using a valid GraphQL API definition with OAuth2 mandatory mTLS optional - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls_optional_oauth2_mandatory.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - And I have a valid token with a client certificate "invalid-cert.txt" - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 401 response code, not accepting - | 429 | - | 500 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-mtls-optional" - Then the response status code should be 202 - - Scenario: Deploying GraphQL API with OAuth2 disabled - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_disabled_auth.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" - And I eventually receive 200 response code, not accepting - | 429 | - | 500 | - And the response body should contain "\"name\":\"string\"" - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-auth-disabled" - Then the response status code should be 202 - - Scenario: Deploying APK conf using a valid GraphQL API definition containing a subscription resource - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_conf_with_sub.apk-conf" - And the definition file "artifacts/definitions/graphql_sample_api.graphql" - And make the API deployment request - Then the response status code should be 200 - - Scenario: Generating APK conf using an invalid GraphQL API definition - Given The system is ready - When I use the definition file "artifacts/definitions/invalid_graphql_api.graphql" in resources - And generate the APK conf file for a "GRAPHQL" API - Then the response status code should be 400 - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "graphql-with-sub" - Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature deleted file mode 100644 index 75a9ee0705..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature +++ /dev/null @@ -1,84 +0,0 @@ -Feature: Test HTTPRoute Filter Header Modifier functionality - Scenario: Test request and response header modification functionality - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" - And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" - And the response body should not contain "\"Authorization\"" - Then the response headers contains key "Set-Response-Header" and value "Test-Value" - Then the response headers contains key "Test-Response-Header" and value "Test-Value" - And the response headers should not contain - | content-type | - - Scenario: Undeploy the API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "api-with-header-modifier-filters" - Then the response status code should be 202 - - Scenario: Test request and response header modification functionality - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-header.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" - And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" - And the response body should not contain "\"Authorization\"" - Then the response headers contains key "Set-Response-Header" and value "Test-Value" - Then the response headers contains key "Test-Response-Header" and value "Test-Value" - And the response headers should not contain - | content-type | - And I send "POST" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" - And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" - And the response body should not contain "\"Authorization\"" - Then the response headers contains key "Set-Response-Header" and value "Test-Value" - Then the response headers contains key "Test-Response-Header" and value "Test-Value" - And the response headers should not contain - | content-type | - And I send "PUT" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" - And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" - And the response body should not contain "\"Authorization\"" - Then the response headers contains key "Set-Response-Header" and value "Test-Value" - Then the response headers contains key "Test-Response-Header" and value "Test-Value" - And the response headers should not contain - | content-type | - And I send "DELETE" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" - And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" - And the response body should not contain "\"Authorization\"" - Then the response headers contains key "Set-Response-Header" and value "Test-Value" - Then the response headers contains key "Test-Response-Header" and value "Test-Value" - And the response headers should not contain - | content-type | - - Scenario: Undeploy the API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "api-with-header-modifier-filters" - Then the response status code should be 202 - - \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature deleted file mode 100644 index 27e5a37fb6..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature +++ /dev/null @@ -1,119 +0,0 @@ -Feature: API Deployment with Interceptor - Scenario: Deploying an API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/interceptors/original.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - And the response body should not contain "\"Interceptor-Header\"" - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestInterceptor.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptor.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation1.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation2.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation3.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation4.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestAndResponse.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestAndResponsetls.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - Scenario Outline: Undeploy an API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | 547961eeaafed989119c45ffc13f8b87bfda821d | 202 | \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/JWT.feature b/test/cucumber-tests/src/test/resources/tests/api/JWT.feature deleted file mode 100644 index 81988e17a2..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/JWT.feature +++ /dev/null @@ -1,126 +0,0 @@ -Feature: Test JWT related functionalities - Scenario: Test JWT authentication with valid and invalid access token - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/jwt_basic_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - Then I set headers - |Authorization|bearer invalidToken| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And the response status code should be 401 - Then I remove header "Authorization" - Then I set headers - | custom-jwt | ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And the response status code should be 200 - Scenario: Test JWT Token from different issuer with JWKS - Given The system is ready - Then I generate JWT token from idp1 with kid "123-456" - Then I set headers - |Authorization|bearer ${idp-1-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - Then I set headers - |Authorization|bearer "${idp-1-token}h"| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |429| - |200| - Then I set headers - |Authorization|bearer ${idp-1-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - Then I generate JWT token from idp1 with kid "456-789" - And I send "DELETE" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/1234" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - Then I set headers - |Authorization|bearer ${idp-1-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |429| - |200| - Scenario: Test disabled JWT configuration - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/jwt_disabled_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer invalidToken| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-disabled/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - - Scenario: Test customized JWT headers - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/jwt_custom_header_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |429| - |200| - Then I set headers - |testAuth|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And the response body should contain "\"X-Jwt-Assertion\"" - And the "X-Jwt-Assertion" jwt should validate from JWKS "https://api.am.wso2.com:9095/.wellknown/jwks" and contain - | claim | value | - | claim1 | value1 | - | claim2 | value2 | - - Scenario: Test customized JWT headers with Resource Endpoint - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/jwt_custom_header_resource_endpoint_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header-resource/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |429| - |200| - Then I set headers - |testAuth|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header-resource/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | jwt-basic-test | 202 | - | jwt-disabled-test | 202 | - | jwt-custom-header-test | 202 | - | jwt-custom-header-resource-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature deleted file mode 100644 index 6cd8fb4fe5..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature +++ /dev/null @@ -1,358 +0,0 @@ -Feature: Test mTLS between client and gateway with client certificate sent in header - # mTLS mandatory OAuth2 mandatory - Scenario: Test mandatory mTLS and mandatory OAuth2 with a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and mandatory OAuth2 with an invalid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "invalid-cert.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and mandatory OAuth2 without a client certificate in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" - Then the response status code should be 202 - - # mTLS optional OAuth2 mandatory - Scenario: Test optional mTLS and mandatory OAuth2 with a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" - Then the response status code should be 202 - - Scenario: Test optional mTLS and mandatory OAuth2 without a token - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" - Then the response status code should be 202 - - Scenario: Test optional mTLS and mandatory OAuth2 with an invalid token in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" - Then the response status code should be 202 - - # mTLS optional OAuth2 disabled - Scenario: Test optional mTLS and disabled OAuth2 with a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_disabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-disabled" - Then the response status code should be 202 - - # mTLS disabled OAuth2 optional - Scenario: Test an API with mTLS disabled and OAuth2 optional - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-disabled-oauth2-optional" - Then the response status code should be 202 - - # mTLS disabled OAuth2 disabled - Scenario: Test an API with mTLS disabled and OAuth2 disabled - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_disabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-disabled-oauth2-disabled" - Then the response status code should be 202 - - # mTLS mandatory OAuth2 disabled - Scenario: Test mandatory mTLS and disabled OAuth2 with a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and disabled OAuth2 with an invalid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "invalid-cert.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and disabled OAuth2 without a client certificate in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" - Then the response status code should be 202 - - # mTLS disabled OAuth2 mandatory - Scenario: Test an API with mTLS disabled and OAuth2 mandatory - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-disabled-oauth2-enabled" - Then the response status code should be 202 - - # Multiple certificates test cases - Scenario: Test an API with mTLS enabled and one associated certificate with multiple certificates existing in system - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And I have a valid token with a client certificate "config-map-2.txt" - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - And I have a valid token with a client certificate "config-map-3.txt" - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" - Then the response status code should be 202 - - Scenario: Test an API with mTLS enabled and multiple certificates configured - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_multiple_certs.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And I have a valid token with a client certificate "config-map-2.txt" - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - And I have a valid token with a client certificate "config-map-3.txt" - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-multiple-certs " - Then the response status code should be 202 diff --git a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature deleted file mode 100644 index e17fd60308..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature +++ /dev/null @@ -1,180 +0,0 @@ -Feature: Test mTLS between client and gateway with client certificate sent in header with OAuth2 optional - # mTLS mandatory OAuth2 optional - Scenario: Test mandatory mTLS and optional OAuth2 with a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and optional OAuth2 with an invalid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "invalid-cert.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test mandatory mTLS and optional OAuth2 without a client certificate in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" - Then the response status code should be 202 - - # mTLS optional OAuth2 optional - Scenario: Test optional mTLS and optional OAuth2 with a valid token and then a valid client certificate in header - Given The system is ready - And I have a valid token with a client certificate "config-map-1.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test optional mTLS and optional OAuth2 with an invalid client certificate and invalid token in header - Given The system is ready - And I have a valid token with a client certificate "invalid-cert.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - | Authorization | bearer {accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test optional mTLS and optional OAuth2 with an invalid client certificate and valid token in header - Given The system is ready - And I have a valid token with a client certificate "invalid-cert.txt" - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - Then I set headers - | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test optional mTLS and optional OAuth2 with an invalid token in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer invalidToken | - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-optional" - Then the response status code should be 202 - - Scenario: Test optional mTLS and optional OAuth2 with no client certificate or token in header - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - | 200 | - - Scenario: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "mtls-optional-oauth2-optional" - Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature b/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature deleted file mode 100644 index 7c49dd524b..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature +++ /dev/null @@ -1,117 +0,0 @@ -Feature: Deploy APIs in multiple environments - Scenario: Deploying an API without specifing an Environment and token issuer has no environments. - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I undeploy the API whose ID is "without-env-api" - Then the response status code should be 202 - - Scenario: Deploying an API without specifing an Environment and token issuer has all(*) environments. - Given The system is ready - And I have a valid token for organization "org3" - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org3" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org3}| - And I send "GET" request to "https://org3.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I undeploy the API whose ID is "without-env-api" and organization "org3" - Then the response status code should be 202 - - Scenario: Deploying an API without specifing an Environment and token issuer has only dev environment. - Given The system is ready - And I have a valid token for organization "org4" - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org4" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org4}| - And I send "GET" request to "https://org4.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |200| - When I undeploy the API whose ID is "without-env-api" and organization "org4" - Then the response status code should be 202 - - Scenario: Deploying APIs in Dev and QA environments and token issuer has no environments. - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_dev.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default-dev.gw.wso2.com:9095/multienv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I undeploy the API whose ID is "multi-env-dev-api" - Then the response status code should be 202 - When I undeploy the API whose ID is "multi-env-qa-api" - Then the response status code should be 202 - - Scenario: Deploying an API in QA environment and token issuer has all(*) environments. - Given The system is ready - And I have a valid token for organization "org3" - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org3" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org3}| - And I send "GET" request to "https://org3-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |401| - When I undeploy the API whose ID is "multi-env-qa-api" and organization "org3" - Then the response status code should be 202 - - Scenario: Deploying an API in QA environment and token issuer has only Dev environment. - Given The system is ready - And I have a valid token for organization "org4" - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org4" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org4}| - And I send "GET" request to "https://org4-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" - And I eventually receive 401 response code, not accepting - |200| - When I undeploy the API whose ID is "multi-env-qa-api" and organization "org4" - Then the response status code should be 202 - - Scenario: Deploying an API in Dev environment and token issuer has only Dev environment. - Given The system is ready - And I have a valid token for organization "org4" - When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_dev.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org4" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org4}| - And I send "GET" request to "https://org4-dev.gw.wso2.com:9095/multienv/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |401| - When I undeploy the API whose ID is "multi-env-dev-api" and organization "org4" - Then the response status code should be 202 - \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature b/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature deleted file mode 100644 index 973ec45068..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature +++ /dev/null @@ -1,47 +0,0 @@ -Feature: Organization Base API Deployment - Scenario: Deploying an API and basic http method invocations - Given The system is ready - And I have a valid token for organization "org1" - When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org1" - Then the response status code should be 200 - And I have a valid token for organization "org2" - When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request for organization "org2" - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${org1}| - And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - Then I set headers - |Authorization|bearer ${org2}| - And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I undeploy the API whose ID is "432bf873bf751ad714ddc635b0a4d9d194b39eb3" and organization "org1" - Then the response status code should be 202 - Then I set headers - |Authorization|bearer ${org1}| - And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 404 response code, not accepting - |429| - Then I set headers - |Authorization|bearer ${org2}| - And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - When I undeploy the API whose ID is "d45e79bec8c4b2d3a7543e09530e0a995ea68691" and organization "org2" - Then the response status code should be 202 - Then I set headers - |Authorization|bearer ${org1}| - And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 404 response code, not accepting - |429| - Then I set headers - |Authorization|bearer ${org2}| - And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" - And I eventually receive 404 response code, not accepting - |429| \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature deleted file mode 100644 index 3dfe4f6af3..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature +++ /dev/null @@ -1,21 +0,0 @@ -Feature: Test HTTPRoute Filter Request Mirror functionality - Scenario: Test request mirror functionality - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-mirror-filter.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/request-mirror-filter/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy the API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "api-with-request-mirror-filter" - Then the response status code should be 202 - - \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature deleted file mode 100644 index 28570f428d..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature +++ /dev/null @@ -1,51 +0,0 @@ -Feature: Test HTTPRoute Filter Request Redirect functionality - Scenario: Test request redirect functionality - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-redirect-filter.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" - And I eventually receive 301 response code, not accepting - | 401 | - And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - | 401 | - - Scenario: Undeploy the API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "api-with-request-redirect-filter" - Then the response status code should be 202 - - Scenario: Test request redirect functionality with API level redirect - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - | Authorization | bearer ${accessToken} | - And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" - And I eventually receive 301 response code, not accepting - | 401 | - And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" - And I eventually receive 301 response code, not accepting - | 401 | - And I send "PUT" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" - And I eventually receive 301 response code, not accepting - | 401 | - And I send "DELETE" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" - And I eventually receive 301 response code, not accepting - | 401 | - - Scenario: Undeploy the API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "api-with-request-redirect-filter" - Then the response status code should be 202 - \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature b/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature deleted file mode 100644 index b4c1adc3a4..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature +++ /dev/null @@ -1,35 +0,0 @@ -Feature: Token revocation - Scenario: Testing token revocation - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/jwt_basic_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And the response status code should be 200 - # Revoke the token - Then I set headers - |stsAuthKey|2jrmypak391zsqz974ugdddebf812ofx1b9t1oq27530ir02tc815eemrx435qvcp41ucgy7v5uuawzi4qcmjrx0k1zgox2s28cr| - And I send "POST" request to "https://api.am.wso2.com:9095/api/notification/1.0.0/notify?type=TOKEN_REVOCATION" with body "{\"token\": \"${accessToken}\"}" - And the response status code should be 200 - And I wait for 5 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" - And the response status code should be 401 - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | jwt-basic-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature b/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature deleted file mode 100644 index 6aa4ca1d2a..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature +++ /dev/null @@ -1,118 +0,0 @@ -Feature: Semantic Versioning Based Intelligent Routing - - Scenario: API version with Major and Minor - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-0.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| - - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.0/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.0\"" - - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.0\"" - - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-1.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.1\"" - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.1\"" - - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-5.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.5/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.5\"" - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.5\"" - - When I undeploy the API whose ID is "sem-api-v1-5" - Then the response status code should be 202 - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - And the response body should contain "\"version\":\"v1.1\"" - - When I undeploy the API whose ID is "sem-api-v1-1" - Then the response status code should be 202 - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - And the response body should contain "\"version\":\"v1.0\"" - - When I undeploy the API whose ID is "sem-api-v1-0" - Then the response status code should be 202 - - Scenario: Multiple Major and minor versions for an API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-0.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-1.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-5.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" - Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.5\"" - - When I undeploy the API whose ID is "sem-api-v1-1" - Then the response status code should be 202 - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.5\"" - - When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v2-1.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v2.1\"" - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2.1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v2.1\"" - - When I undeploy the API whose ID is "sem-api-v1-0" - Then the response status code should be 202 - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v1.5\"" - - When I undeploy the API whose ID is "sem-api-v1-5" - Then the response status code should be 202 - And I wait for 2 seconds - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" - Then the response status code should be 404 - - And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2/employee/" with body "" - Then the response status code should be 200 - And the response body should contain "\"version\":\"v2.1\"" - - When I undeploy the API whose ID is "sem-api-v2-1" - Then the response status code should be 202 diff --git a/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature b/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature deleted file mode 100644 index 7b88660c50..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature +++ /dev/null @@ -1,76 +0,0 @@ -Feature: Test simple rate limit feature - Scenario: Test simple rate limit api level - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/simple_rl_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I wait for next minute - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" - Then the response status code should be 429 - And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" - Then the response status code should be 429 - - Scenario: Test simple rate limit api level for unsecured api - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/simple_rl_jwt_disabled_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|invalid| - And I wait for next minute - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-jwt-disabled/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-jwt-disabled/3.14/employee/" with body "" - Then the response status code should be 429 - - Scenario: Test simple rate limit resource level - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/simple_rl_resource_conf.yaml" - And the definition file "artifacts/definitions/employees_api.json" - And make the API deployment request - Then the response status code should be 200 - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "POST" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "POST" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - Then the response status code should be 429 - And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - And I eventually receive 200 response code, not accepting - |429| - |401| - And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - Then the response status code should be 429 - And I wait for next minute - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - Then the response status code should be 200 - And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" - Then the response status code should be 200 - - - Scenario Outline: Undeploy API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | simple-rl-test | 202 | - | simple-rl-r-test | 202 | - | simple-rl-jwt-disabled-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/deployment.feature b/test/cucumber-tests/src/test/resources/tests/api/deployment.feature deleted file mode 100644 index 56e2ef2605..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/deployment.feature +++ /dev/null @@ -1,44 +0,0 @@ -Feature: API Deployment - Scenario: Deploying an API without api create scope - Given The system is ready - And I have a valid subscription without api deploy permission - When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 403 - - Scenario: Deploying an API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "cors-api-adff3dbc-2787-11ee-be56-0242ac120002" - - Scenario: Deploying an API with invalid APK Conf file - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/invalid_cors_API.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 400 - And the response body should contain - |"#/corsConfiguration/corsConfigurationEnabled: expected type: Boolean, found: String"| - - Scenario Outline: Undeploy an API without api create scope - Given The system is ready - And I have a valid subscription without api deploy permission - When I undeploy the API whose ID is "" - Then the response status code should be 403 - - Scenario Outline: Undeploy an API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | cors-api-adff3dbc-2787-11ee-be56-0242ac120002 | 202 | - | abcdeadsxzads | 404 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature deleted file mode 100644 index 93271f5287..0000000000 --- a/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature +++ /dev/null @@ -1,34 +0,0 @@ -Feature: API Deployment with Resource Interceptor - Scenario: Deploying an API - Given The system is ready - And I have a valid subscription - When I use the APK Conf file "artifacts/apk-confs/interceptors/resourceLevelInterptor.apk-conf" - And the definition file "artifacts/definitions/cors_api.yaml" - And make the API deployment request - Then the response status code should be 200 - And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" - And I wait for 1 minute - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" - And the response body should not contain "\"Interceptor-Header\"" - Then the response status code should be 200 - Then the response headers not contains key "interceptor-response-header" - Then I set headers - |Authorization|bearer ${accessToken}| - And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/headers" with body "" - And the response body should contain - |"Interceptor-Header": "Interceptor-header-value"| - |"Interceptor-Header-Apigroup": "Gold"| - |"Interceptor-Header-Apitier": "Unlimited"| - Then the response status code should be 200 - Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" - Scenario Outline: Undeploy an API - Given The system is ready - And I have a valid subscription - When I undeploy the API whose ID is "" - Then the response status code should be - - Examples: - | apiID | expectedStatusCode | - | 547961eeaafed989119c45ffc13f8b87bfda821d | 202 | \ No newline at end of file From b21fcf23af2a7c0bf6b892384a0c03159baabd86 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Mon, 9 Sep 2024 14:24:25 +0530 Subject: [PATCH 02/24] Fix rl key not deleted bug --- adapter/internal/operator/controllers/dp/api_controller.go | 2 +- common-controller/internal/xds/server.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 30ce6a4918..ede19399bd 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -1486,7 +1486,7 @@ func (apiReconciler *APIReconciler) getAPIsForAIRatelimitPolicy(ctx context.Cont } if aiRatelimitPolicy.Spec.TargetRef.Kind == constants.KindBackend { - backend := &dpv1alpha1.Backend{} + backend := &dpv1alpha2.Backend{} namespacedName := types.NamespacedName{ Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), Namespace: utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()), diff --git a/common-controller/internal/xds/server.go b/common-controller/internal/xds/server.go index 457f754d42..ec4ef38a0c 100644 --- a/common-controller/internal/xds/server.go +++ b/common-controller/internal/xds/server.go @@ -167,11 +167,7 @@ func UpdateRateLimitXDSCacheForCustomPolicies(customRateLimitPolicies dpv1alpha1 // UpdateRateLimitXDSCacheForAIRatelimitPolicies updates the xDS cache of the RateLimiter for AI ratelimit policies. func UpdateRateLimitXDSCacheForAIRatelimitPolicies(aiRatelimitPolicySpecs map[apimachiner_types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { - loggers.LoggerAPKOperator.Infof("000000") - if len(aiRatelimitPolicySpecs) != 0 { - loggers.LoggerAPKOperator.Infof("1111111") - rlsPolicyCache.ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRatelimitPolicySpecs) - } + rlsPolicyCache.ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRatelimitPolicySpecs) } // DeleteAPILevelRateLimitPolicies delete the ratelimit xds cache From 98442245acc1d045efddb417a814ebc39e2bf474 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Mon, 9 Sep 2024 14:39:39 +0530 Subject: [PATCH 03/24] Fix build --- common-controller/internal/xds/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/common-controller/internal/xds/server.go b/common-controller/internal/xds/server.go index ec4ef38a0c..f1aa55c271 100644 --- a/common-controller/internal/xds/server.go +++ b/common-controller/internal/xds/server.go @@ -33,7 +33,6 @@ import ( wso2_cache "github.com/wso2/apk/adapter/pkg/discovery/protocol/cache/v3" eventhubTypes "github.com/wso2/apk/adapter/pkg/eventhub/types" - "github.com/wso2/apk/common-controller/internal/loggers" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" apimachiner_types "k8s.io/apimachinery/pkg/types" From 5e4a3089ad7eaa1dc2339f08c13c2e42b7289ab4 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Mon, 9 Sep 2024 20:24:02 +0530 Subject: [PATCH 04/24] add subscription based ai ratelimit functionality to adaptetr and cc --- .../oasparser/envoyconf/routes_configs.go | 158 +++++++++++++++--- .../operator/controllers/dp/api_controller.go | 71 +++----- adapter/internal/operator/operator.go | 2 + common-controller/internal/cache/datastore.go | 44 ++++- .../controllers/cp/subscription_controller.go | 77 ++++++++- .../dp/airatelimitpolicy_controller.go | 12 +- .../internal/operator/operator.go | 2 +- .../internal/xds/ratelimiter_cache.go | 113 +++++++++++-- common-controller/internal/xds/server.go | 5 + 9 files changed, 386 insertions(+), 98 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index 797193e7ad..8913a1e5d0 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -44,10 +44,6 @@ import ( v35 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" ) -const( - authzNamespace = "envoy.filters.http.ext_authz" -) - // Constants for Rate Limiting const ( DescriptorKeyForSubscription = "subscription" @@ -61,6 +57,25 @@ const ( descriptorMetadataKeyForBurstCtrlSubscription = "burstCtrl:subscription" descriptorMetadataKeyForBurstCtrlUsagePolicy = "burstCtrl:usage-policy" descriptorMetadataKeyForBurstCtrlOrganization = "burstCtrl:organization" + // DescriptorKeyForAIRequestTokenCount is the descriptor key for AI request token count ratelimit + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + // DescriptorKeyForAIResponseTokenCount is the descriptor key for AI response token count ratelimit + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + // DescriptorKeyForAITotalTokenCount is the descriptor key for AI total token count ratelimit + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + // DescriptorKeyForAIRequestCount is the descriptor key for AI request count ratelimit + DescriptorKeyForAIRequestCount = "airequestcount" + // DescriptorKeyForAIRequestTokenCountForSubscriptionBasedAIRL is the descriptor key for AI request token count ratelimit + DescriptorKeyForAIRequestTokenCountForSubscriptionBasedAIRL = "airequesttokencountsubs" + // DescriptorKeyForAIResponseTokenCountForSubscriptionBasedAIRL is the descriptor key for AI response token count ratelimit + DescriptorKeyForAIResponseTokenCountForSubscriptionBasedAIRL = "airesponsetokencountsubs" + // DescriptorKeyForAITotalTokenCountForSubscriptionBasedAIRL is the descriptor key for AI total token count ratelimit + DescriptorKeyForAITotalTokenCountForSubscriptionBasedAIRL = "aitotaltokencountsubs" + // DescriptorKeyForAIRequestCountForSubscriptionBasedAIRL is the descriptor key for AI request count ratelimit + DescriptorKeyForAIRequestCountForSubscriptionBasedAIRL = "airequestcountsubs" + DynamicMetadataKeyForOrganizationAndAIRLPolicy = "ratelimit:organization-and-rlpolicy" + DynamicMetadataKeyForSubscription = "ratelimit:subscription" + DescriptorKeyForAISubscription = "subscription" ) func generateRouteConfig(routeName string, match *routev3.RouteMatch, action *routev3.Route_Route, redirectAction *routev3.Route_Redirect, @@ -191,16 +206,6 @@ func mapStatusCodeToEnum(statusCode int) int { return -1 } } -const ( - // DescriptorKeyForAIRequestTokenCount is the descriptor key for AI request token count ratelimit - DescriptorKeyForAIRequestTokenCount = "airequesttokencount" - // DescriptorKeyForAIResponseTokenCount is the descriptor key for AI response token count ratelimit - DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" - // DescriptorKeyForAITotalTokenCount is the descriptor key for AI total token count ratelimit - DescriptorKeyForAITotalTokenCount = "aitotaltokencount" - // DescriptorKeyForAIRequestCount is the descriptor key for AI request count ratelimit - DescriptorKeyForAIRequestCount = "airequestcount" -) func generateBackendBasedAIRatelimit(descValue string) []*routev3.RateLimit { rateLimitForRequestTokenCount := routev3.RateLimit{ @@ -261,17 +266,38 @@ func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { ActionSpecifier: &routev3.RateLimit_Action_Metadata{ Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIRequestTokenCount, + DescriptorKey: DescriptorKeyForAIRequestTokenCountForSubscriptionBasedAIRL, + MetadataKey: &v35.MetadataKey{ + Key: extAuthzFilterName, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, MetadataKey: &v35.MetadataKey{ - Key: authzNamespace, + Key: extAuthzFilterName, Path: []*v35.MetadataKey_PathSegment{ &v35.MetadataKey_PathSegment{ Segment: &v35.MetadataKey_PathSegment_Key{ - Key: "", + Key: DynamicMetadataKeyForSubscription, }, }, }, }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, }, }, }, @@ -282,17 +308,38 @@ func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { ActionSpecifier: &routev3.RateLimit_Action_Metadata{ Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIResponseTokenCount, + DescriptorKey: DescriptorKeyForAIResponseTokenCountForSubscriptionBasedAIRL, + MetadataKey: &v35.MetadataKey{ + Key: extAuthzFilterName, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, MetadataKey: &v35.MetadataKey{ - Key: authzNamespace, + Key: extAuthzFilterName, Path: []*v35.MetadataKey_PathSegment{ &v35.MetadataKey_PathSegment{ Segment: &v35.MetadataKey_PathSegment_Key{ - Key: "", + Key: DynamicMetadataKeyForSubscription, }, }, }, }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, }, }, }, @@ -303,23 +350,86 @@ func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { ActionSpecifier: &routev3.RateLimit_Action_Metadata{ Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIRequestCount, + DescriptorKey: DescriptorKeyForAIRequestCountForSubscriptionBasedAIRL, MetadataKey: &v35.MetadataKey{ - Key: authzNamespace, + Key: extAuthzFilterName, Path: []*v35.MetadataKey_PathSegment{ &v35.MetadataKey_PathSegment{ Segment: &v35.MetadataKey_PathSegment_Key{ - Key: "", + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, }, }, }, }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &v35.MetadataKey{ + Key: extAuthzFilterName, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, }, }, }, }, } - return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount} + rateLimitForTotalTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAITotalTokenCountForSubscriptionBasedAIRL, + MetadataKey: &v35.MetadataKey{ + Key: extAuthzFilterName, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &v35.MetadataKey{ + Key: extAuthzFilterName, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount, &rateLimitForTotalTokenCount} } func generateRateLimitPolicy(ratelimitCriteria *ratelimitCriteria) []*routev3.RateLimit { diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index ede19399bd..09c3bd1c4a 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -58,7 +58,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" k8client "sigs.k8s.io/controller-runtime/pkg/client" - cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" + cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" @@ -97,7 +97,6 @@ const ( backendInterceptorServiceIndex = "backendInterceptorServiceIndex" backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" aiRatelimitPolicyToBackendIndex = "aiRatelimitPolicyToBackendIndex" - aiRatelimitPolicyToSubscriptionIndex = "aiRatelimitPolicyToSubscriptionIndex" subscriptionToAPIIndex = "subscriptionToAPIIndex" apiToSubscriptionIndex = "apiToSubscriptionIndex" aiProviderAPIPolicyIndex = "aiProviderAPIPolicyIndex" @@ -233,7 +232,7 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera return err } - if err := c.Watch(source.Kind(mgr.GetCache(), &cpv1alpha2.Subscription{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForSubscription), + if err := c.Watch(source.Kind(mgr.GetCache(), &cpv1alpha3.Subscription{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForSubscription), predicates...); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching Subscription resources: %v", err)) return err @@ -857,7 +856,7 @@ func (apiReconciler *APIReconciler) getAPIPolicyChildrenRefs(ctx context.Context func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx context.Context, apiState *synchronizer.APIState) { apiState.IsAiSubscriptionRatelimitEnabled = false - subscriptionList := &cpv1alpha2.SubscriptionList{} + subscriptionList := &cpv1alpha3.SubscriptionList{} if err := apiReconciler.client.List(ctx, subscriptionList, &k8client.ListOptions{ FieldSelector: fields.OneTermEqualSelector(subscriptionToAPIIndex, utils.GetSubscriptionToAPIIndexID(apiState.APIDefinition.Spec.APIName, apiState.APIDefinition.Spec.APIVersion)), }); err != nil { @@ -865,16 +864,13 @@ func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx c return } for _, subscription := range subscriptionList.Items { - aiRatelimitPolicyList := &dpv1alpha3.AIRateLimitPolicyList{} - if err := apiReconciler.client.List(ctx, aiRatelimitPolicyList, &k8client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToSubscriptionIndex, utils.NamespacedName(&subscription).String()), - }); err != nil { + aiRatelimitPolicy := &dpv1alpha3.AIRateLimitPolicy{} + if err := apiReconciler.client.Get(ctx, utils.NamespacedName(&subscription), aiRatelimitPolicy, ); err != nil { loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) return } - if len(aiRatelimitPolicyList.Items) > 0 { - apiState.IsAiSubscriptionRatelimitEnabled = true - } + apiState.IsAiSubscriptionRatelimitEnabled = true + return } } @@ -897,8 +893,6 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte // Resolve backends in HTTPRoute httpRoute := httpRouteState.HTTPRouteCombined - // httpRouteState.AiRatelimit_HttpRouteRulesMapping = aiRatelimitPolicy_routeRulematching - // httpRouteState.AiRatelimitPolicyMapping = aiRatelimitPolicyMapping ruleIdxToAiRatelimitPolicyMapping := make(map[int]*dpv1alpha3.AIRateLimitPolicy) httpRouteState.RuleIdxToAiRatelimitPolicyMapping = ruleIdxToAiRatelimitPolicyMapping for id, rule := range httpRoute.Spec.Rules { @@ -916,23 +910,6 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte for _, aiRLPolicy := range aiRLPolicyList.Items { loggers.LoggerAPKOperator.Infof("Adding mapping for ruleid: %d to aiRLPolicy: %s", id, utils.NamespacedName(&aiRLPolicy)) ruleIdxToAiRatelimitPolicyMapping[id] = &aiRLPolicy - // aiRlPolicyNN := utils.NamespacedName(&aiRLPolicy) - // if ruleList, exists := aiRatelimitPolicy_routeRulematching[aiRlPolicyNN]; !exists { - // ruleListNew := make([]*gwapiv1.HTTPRouteRule, 0) - // ruleListNew = append(ruleListNew, &rule) - // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleListNew - // } else { - // ruleList = append(ruleList, &rule) - // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleList - // } - // if aiRLList, exists := aiRatelimitPolicyMapping[aiRlPolicyNN]; !exists { - // aiRlListNew := make([]*v1alpha3.AIRateLimitPolicy, 0) - // aiRlListNew = append(aiRlListNew, &aiRLPolicy) - // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRlListNew - // } else { - // aiRLList = append(aiRLList, &aiRLPolicy) - // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRLList - // } } } if _, exists := backendMapping[backendNamespacedName.String()]; !exists { @@ -1504,7 +1481,7 @@ func (apiReconciler *APIReconciler) getAPIsForAIRatelimitPolicy(ctx context.Cont // getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected // in subscription resources. func (apiReconciler *APIReconciler) getAPIsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { - subscription, ok := obj.(*cpv1alpha2.Subscription) + subscription, ok := obj.(*cpv1alpha3.Subscription) if !ok { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) return []reconcile.Request{} @@ -2101,25 +2078,25 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } - // AIRatelimitPolicy to Subscription indexer - if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToSubscriptionIndex, - func(rawObj k8client.Object) []string { - aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) - var subscriptions []string - namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) - subscriptions = append(subscriptions, types.NamespacedName{ - Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), - Namespace: namespace, - }.String()) - return subscriptions - }); err != nil { - return err - } + // // AIRatelimitPolicy to Subscription indexer + // if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToSubscriptionIndex, + // func(rawObj k8client.Object) []string { + // aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) + // var subscriptions []string + // namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) + // subscriptions = append(subscriptions, types.NamespacedName{ + // Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + // Namespace: namespace, + // }.String()) + // return subscriptions + // }); err != nil { + // return err + // } // Subscription to API indexer - if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha2.Subscription{}, subscriptionToAPIIndex, + if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha3.Subscription{}, subscriptionToAPIIndex, func(rawObj k8client.Object) []string { - subscription := rawObj.(*cpv1alpha2.Subscription) + subscription := rawObj.(*cpv1alpha3.Subscription) var subscriptions []string subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", subscription.Spec.API.Name, subscription.Spec.API.Version) subscriptions = append(subscriptions, subscriptionIdentifierForIndex) diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index ece0e47e1a..b7236ce5ed 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -51,6 +51,7 @@ import ( dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" + cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" //+kubebuilder:scaffold:imports ) @@ -72,6 +73,7 @@ func init() { utilruntime.Must(dpv1alpha3.AddToScheme(scheme)) utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) + utilruntime.Must(cpv1alpha3.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/common-controller/internal/cache/datastore.go b/common-controller/internal/cache/datastore.go index 293a3c9e3c..b21ba84b74 100644 --- a/common-controller/internal/cache/datastore.go +++ b/common-controller/internal/cache/datastore.go @@ -29,11 +29,12 @@ import ( // RatelimitDataStore is a cache for rate limit policies. type RatelimitDataStore struct { - resolveRatelimitStore map[types.NamespacedName][]dpv1alpha1.ResolveRateLimitAPIPolicy - resolveSubscriptionRatelimitStore map[types.NamespacedName]dpv1alpha3.ResolveSubscriptionRatelimitPolicy - customRatelimitStore map[types.NamespacedName]*dpv1alpha1.CustomRateLimitPolicyDef - mu sync.Mutex - aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec + resolveRatelimitStore map[types.NamespacedName][]dpv1alpha1.ResolveRateLimitAPIPolicy + resolveSubscriptionRatelimitStore map[types.NamespacedName]dpv1alpha3.ResolveSubscriptionRatelimitPolicy + customRatelimitStore map[types.NamespacedName]*dpv1alpha1.CustomRateLimitPolicyDef + mu sync.Mutex + aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec + subscriptionEnabledAIRatelimitPolicies map[types.NamespacedName]struct{} } // CreateNewOperatorDataStore creates a new RatelimitDataStore. @@ -102,6 +103,26 @@ func (ods *RatelimitDataStore) AddorUpdateAIRatelimitToStore(rateLimit types.Nam ods.aiRatelimitPolicySpecs[rateLimit] = &aiRatelimitSpec } +// MarkAIRatelimitAsSubscriptionEnabled +func (ods *RatelimitDataStore) MarkAIRatelimitAsSubscriptionEnabled(nn types.NamespacedName) { + ods.mu.Lock() + defer ods.mu.Unlock() + if ods.subscriptionEnabledAIRatelimitPolicies == nil { + ods.subscriptionEnabledAIRatelimitPolicies = make(map[types.NamespacedName]struct{}) + } + ods.subscriptionEnabledAIRatelimitPolicies[nn] = struct{}{} +} + +// MarkAIRatelimitAsSubscriptionDisabled +func (ods *RatelimitDataStore) MarkAIRatelimitAsSubscriptionDisabled(nn types.NamespacedName) { + ods.mu.Lock() + defer ods.mu.Unlock() + if ods.subscriptionEnabledAIRatelimitPolicies == nil { + return + } + delete(ods.subscriptionEnabledAIRatelimitPolicies, nn) +} + // GetResolveRatelimitPolicy get cached ratelimit func (ods *RatelimitDataStore) GetResolveRatelimitPolicy(rateLimit types.NamespacedName) ([]dpv1alpha1.ResolveRateLimitAPIPolicy, bool) { var rateLimitPolicy []dpv1alpha1.ResolveRateLimitAPIPolicy @@ -127,6 +148,11 @@ func (ods *RatelimitDataStore) GetAIRatelimitPolicySpecs() map[types.NamespacedN return ods.aiRatelimitPolicySpecs } +// GetAIRatelimitPolicySpecs gets all the AIRatelimitPolicy stored in ods +func (ods *RatelimitDataStore) GetSubscriptionEnabledAIRatelimitPolicies() map[types.NamespacedName]struct{} { + return ods.subscriptionEnabledAIRatelimitPolicies +} + // DeleteResolveRatelimitPolicy delete from ratelimit cache func (ods *RatelimitDataStore) DeleteResolveRatelimitPolicy(rateLimit types.NamespacedName) { ods.mu.Lock() @@ -151,6 +177,14 @@ func (ods *RatelimitDataStore) DeleteAIRatelimitPolicySpec(rateLimit types.Names delete(ods.aiRatelimitPolicySpecs, rateLimit) } +// DeleteSubscriptionBasedAIRatelimitPolicySpec delete from ratelimit cache +func (ods *RatelimitDataStore) DeleteSubscriptionBasedAIRatelimitPolicySpec(subscription types.NamespacedName) { + ods.mu.Lock() + defer ods.mu.Unlock() + logger.Debug("Deleting AI ratelimit from cache") + delete(ods.aiRatelimitPolicySpecs, subscription) +} + // NamespacedName generates namespaced name for Kubernetes objects func NamespacedName(obj client.Object) types.NamespacedName { return types.NamespacedName{ diff --git a/common-controller/internal/operator/controllers/cp/subscription_controller.go b/common-controller/internal/operator/controllers/cp/subscription_controller.go index 5d8212474b..9167095af0 100644 --- a/common-controller/internal/operator/controllers/cp/subscription_controller.go +++ b/common-controller/internal/operator/controllers/cp/subscription_controller.go @@ -44,6 +44,7 @@ import ( cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + xds "github.com/wso2/apk/common-controller/internal/xds" ) // SubscriptionReconciler reconciles a Subscription object @@ -51,17 +52,20 @@ type SubscriptionReconciler struct { client client.Client Scheme *runtime.Scheme ods *cache.SubscriptionDataStore + rlODS *cache.RatelimitDataStore } const ( - subscriptionRatelimitIndex = "subscriptionRatelimitIndex" + subscriptionRatelimitIndex = "subscriptionRatelimitIndex" + subscriptionToAIRatelimitIndex = "subscriptionToAIRatelimitIndex" ) // NewSubscriptionController creates a new Subscription controller instance. -func NewSubscriptionController(mgr manager.Manager, subscriptionStore *cache.SubscriptionDataStore) error { +func NewSubscriptionController(mgr manager.Manager, subscriptionStore *cache.SubscriptionDataStore, ratelimitStore *cache.RatelimitDataStore) error { r := &SubscriptionReconciler{ client: mgr.GetClient(), ods: subscriptionStore, + rlODS: ratelimitStore, } ctx := context.Background() conf := config.ReadConfigs() @@ -88,6 +92,12 @@ func NewSubscriptionController(mgr manager.Manager, subscriptionStore *cache.Sub return err } + if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), handler.EnqueueRequestsFromMapFunc(r.getSubscriptionForAIRatelimit), + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2613, logging.BLOCKER, "Error watching AIratelimit policies resources: %v", err)) + return err + } + loggers.LoggerAPKOperator.Debug("Subscription Controller successfully started. Watching Subscription Objects...") return nil } @@ -125,6 +135,20 @@ func (subscriptionReconciler *SubscriptionReconciler) Reconcile(ctx context.Cont } } } else { + if subscription.Spec.RatelimitRef.Name != "" { + nn := types.NamespacedName{ + Namespace: subscription.Namespace, + Name: subscription.Spec.RatelimitRef.Name, + } + var airl dpv1alpha3.AIRateLimitPolicy + if err := subscriptionReconciler.client.Get(ctx, nn, &airl); err == nil { + subscriptionReconciler.rlODS.AddorUpdateAIRatelimitToStore(nn, airl.Spec) + subscriptionReconciler.rlODS.MarkAIRatelimitAsSubscriptionEnabled(nn) + xds.UpdateRateLimitXDSCacheForAubscriptionBasedAIRatelimitPolicies(subscriptionReconciler.rlODS.GetSubscriptionEnabledAIRatelimitPolicies(), subscriptionReconciler.rlODS.GetAIRatelimitPolicySpecs()) + conf := config.ReadConfigs() + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } + } sendSubUpdates(subscription) utils.SendAddSubscriptionEvent(subscription) subscriptionReconciler.ods.AddorUpdateSubscriptionToStore(subscriptionKey, subscription.Spec) @@ -169,6 +193,20 @@ func addSubscriptionControllerIndexes(ctx context.Context, mgr manager.Manager) loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2610, logging.CRITICAL, "Error adding indexes: %v", err)) return err } + if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha3.Subscription{}, subscriptionToAIRatelimitIndex, + func(rawObj k8client.Object) []string { + subscription := rawObj.(*cpv1alpha3.Subscription) + var aiRatelimits []string + aiRatelimits = append(aiRatelimits, + types.NamespacedName{ + Name: string(subscription.Spec.RatelimitRef.Name), + Namespace: subscription.Namespace, + }.String()) + return aiRatelimits + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2610, logging.CRITICAL, "Error adding indexes: %v", err)) + return err + } return nil } @@ -207,3 +245,38 @@ func (subscriptionReconciler *SubscriptionReconciler) getSubscriptionForRatelimi } return requests } + +// getSubscriptionForAIRatelimit get the associated subscription reconcile request for a AIRatelimit resource change +func (subscriptionReconciler *SubscriptionReconciler) getSubscriptionForAIRatelimit(ctx context.Context, obj k8client.Object) []reconcile.Request { + airatelimit, ok := obj.(*dpv1alpha3.AIRateLimitPolicy) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", airatelimit)) + return []reconcile.Request{} + } + + subList := &cpv1alpha3.SubscriptionList{} + if err := subscriptionReconciler.client.List(ctx, subList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(subscriptionToAIRatelimitIndex, utils.NamespacedName(airatelimit).String()), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2623, logging.CRITICAL, "Unable to find associated AI ratelimits: %s", utils.NamespacedName(airatelimit).String())) + return []reconcile.Request{} + } + + if len(subList.Items) == 0 { + loggers.LoggerAPKOperator.Debugf("AIRatelimit for Subscription %s/%s not found", airatelimit.Namespace, airatelimit.Name) + return []reconcile.Request{} + } + + requests := []reconcile.Request{} + for _, subscription := range subList.Items { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: subscription.Name, + Namespace: subscription.Namespace}, + } + requests = append(requests, req) + loggers.LoggerAPKOperator.Debugf("Adding reconcile request for AIratelimit policy: %s/%s with Subscription UUID: %v", subscription.Namespace, subscription.Name, + string(subscription.ObjectMeta.UID)) + } + return requests +} diff --git a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go index afe263ae8a..0d618a2277 100644 --- a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go +++ b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go @@ -143,9 +143,15 @@ func (r *AIRateLimitPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) } else { loggers.LoggerAPKOperator.Infof("ratelimits found") - r.ods.AddorUpdateAIRatelimitToStore(ratelimitKey, ratelimitPolicy.Spec) - xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) - xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + if ratelimitPolicy.Spec.TargetRef.Name != "" { + r.ods.AddorUpdateAIRatelimitToStore(ratelimitKey, ratelimitPolicy.Spec) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } else { + r.ods.DeleteAIRatelimitPolicySpec(ratelimitKey) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } } loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile..*****.") return ctrl.Result{}, nil diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go index ef2ab5eadc..9a2acd91ae 100644 --- a/common-controller/internal/operator/operator.go +++ b/common-controller/internal/operator/operator.go @@ -176,7 +176,7 @@ func InitOperator(metricsConfig config.Metrics) { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3115, logging.MAJOR, "Error creating Application controller, error: %v", err)) } - if err := cpcontrollers.NewSubscriptionController(mgr, subscriptionStore); err != nil { + if err := cpcontrollers.NewSubscriptionController(mgr, subscriptionStore, ratelimitStore); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3116, logging.MAJOR, "Error creating Subscription controller, error: %v", err)) } diff --git a/common-controller/internal/xds/ratelimiter_cache.go b/common-controller/internal/xds/ratelimiter_cache.go index 9b9d4bb089..7f7d918a10 100644 --- a/common-controller/internal/xds/ratelimiter_cache.go +++ b/common-controller/internal/xds/ratelimiter_cache.go @@ -38,20 +38,25 @@ import ( // Constants relevant to the route related ratelimit configurations const ( - DescriptorKeyForOrg = "org" - OrgMetadataKey = "customorg" - DescriptorKeyForEnvironment = "environment" - DescriptorKeyForPath = "path" - DescriptorKeyForMethod = "method" - DescriptorValueForAPIMethod = "ALL" - DescriptorValueForOperationMethod = ":method" - MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" - MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" - apiDefinitionClusterName = "api_definition_cluster" - DescriptorKeyForAIRequestTokenCount = "airequesttokencount" - DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" - DescriptorKeyForAITotalTokenCount = "aitotaltokencount" - DescriptorKeyForAIRequestCount = "airequestcount" + DescriptorKeyForOrg = "org" + OrgMetadataKey = "customorg" + DescriptorKeyForEnvironment = "environment" + DescriptorKeyForPath = "path" + DescriptorKeyForMethod = "method" + DescriptorValueForAPIMethod = "ALL" + DescriptorValueForOperationMethod = ":method" + MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" + MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" + apiDefinitionClusterName = "api_definition_cluster" + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + DescriptorKeyForAIRequestCount = "airequestcount" + DescriptorKeyForSubscriptionBasedAIRequestTokenCount = "airequesttokencountsubs" + DescriptorKeyForSubscriptionBasedAIResponseTokenCount = "airesponsetokencountsubs" + DescriptorKeyForSubscriptionBasedAITotalTokenCount = "aitotaltokencountsubs" + DescriptorKeyForSubscriptionBasedAIRequestCount = "airequestcountsubs" + DescriptorKeyForSubscription = "subscription" ) const ( @@ -86,7 +91,8 @@ type rateLimitPolicyCache struct { // org -> Custom Rate Limit Configs customRateLimitPolicies map[string]map[string]*rls_config.RateLimitDescriptor - aiRatelimitDescriptors []*rls_config.RateLimitDescriptor + aiRatelimitDescriptors []*rls_config.RateLimitDescriptor + subscriptionBasedAIRatelimitDescriptors []*rls_config.RateLimitDescriptor // mutex for API level apiLevelMu sync.RWMutex @@ -291,6 +297,10 @@ func (r *rateLimitPolicyCache) generateRateLimitConfig() *rls_config.RateLimitCo // Add AI ratelimit descriptors orgDescriptors = append(orgDescriptors, r.aiRatelimitDescriptors...) + + // Add Subscription bases AI ratelimit descriptors + orgDescriptors = append(orgDescriptors, r.subscriptionBasedAIRatelimitDescriptors...) + return &rls_config.RateLimitConfig{ Name: RateLimiterDomain, Domain: RateLimiterDomain, @@ -322,6 +332,74 @@ func (r *rateLimitPolicyCache) AddCustomRateLimitPolicies(customRateLimitPolicy } } +// ProcessSubscriptionBasedAIRatelimitPolicySpecsAndUpdateCache process the specs and update the cache +func (r *rateLimitPolicyCache) ProcessSubscriptionBasedAIRatelimitPolicySpecsAndUpdateCache(subscriptionEnabledAIRatelimitPolicies map[types.NamespacedName]struct{}, aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) + loggers.LoggerAPKOperator.Infof("222222") + for namespacedNameRl := range subscriptionEnabledAIRatelimitPolicies { + if airl, exists := aiRatelimitPolicySpecs[namespacedNameRl]; exists { + loggers.LoggerAPKOperator.Infof("----- %s %s %s", DescriptorKeyForSubscriptionBasedAIRequestTokenCount, prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), DescriptorKeyForSubscription) + // Add descriptor for RequestTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForSubscriptionBasedAIRequestTokenCount, + Value: prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), + Descriptors: []*rls_config.RateLimitDescriptor{ + { + Key: DescriptorKeyForSubscription, + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(airl.Override.TokenCount.Unit), + RequestsPerUnit: uint32(airl.Override.TokenCount.RequestTokenCount), + }, + }, + }, + }) + // Add descriptor for ResponseTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForSubscriptionBasedAIResponseTokenCount, + Value: prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), + Descriptors: []*rls_config.RateLimitDescriptor{ + { + Key: DescriptorKeyForSubscription, + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(airl.Override.TokenCount.Unit), + RequestsPerUnit: uint32(airl.Override.TokenCount.ResponseTokenCount), + }, + }, + }, + }) + // Add descriptor for TotalTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForSubscriptionBasedAITotalTokenCount, + Value: prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), + Descriptors: []*rls_config.RateLimitDescriptor{ + { + Key: DescriptorKeyForSubscription, + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(airl.Override.TokenCount.Unit), + RequestsPerUnit: uint32(airl.Override.TokenCount.TotalTokenCount), + }, + }, + }, + }) + // Add descriptor for RequestCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForSubscriptionBasedAIRequestCount, + Value: prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), + Descriptors: []*rls_config.RateLimitDescriptor{ + { + Key: DescriptorKeyForSubscription, + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(airl.Override.TokenCount.Unit), + RequestsPerUnit: uint32(airl.Override.RequestCount.RequestsPerUnit), + }, + }, + }, + }) + } + } + r.subscriptionBasedAIRatelimitDescriptors = aiRlDescriptors +} + // ProcessAIratelimitPolicySpecsAndUpdateCache process the specs and update the cache func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRateLimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) @@ -369,6 +447,10 @@ func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRat r.aiRatelimitDescriptors = aiRlDescriptors } +func prepareSubscriptionBasedAIRatelimitIdentifier(org string, namespacedName types.NamespacedName) string { + return fmt.Sprintf("%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name)) +} + func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { targetNamespace := string(namespacedName.Namespace) if spec.TargetRef.Namespace != nil && string(*spec.TargetRef.Namespace) != "" { @@ -512,7 +594,6 @@ func parseRateLimitPolicyToXDS(policy dpv1alpha1.ResolveRateLimit) *rls_config.R } func getRateLimitUnit(name string) rls_config.RateLimitUnit { - loggers.LoggerAPKOperator.Info("Rate limit unit: ", name) switch strings.ToUpper(name) { case "SECOND": return rls_config.RateLimitUnit_SECOND diff --git a/common-controller/internal/xds/server.go b/common-controller/internal/xds/server.go index f1aa55c271..ff8d8375d7 100644 --- a/common-controller/internal/xds/server.go +++ b/common-controller/internal/xds/server.go @@ -169,6 +169,11 @@ func UpdateRateLimitXDSCacheForAIRatelimitPolicies(aiRatelimitPolicySpecs map[ap rlsPolicyCache.ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRatelimitPolicySpecs) } +// UpdateRateLimitXDSCacheForAubscriptionBasedAIRatelimitPolicies updates the xDS cache of the RateLimiter for AI ratelimit policies. +func UpdateRateLimitXDSCacheForAubscriptionBasedAIRatelimitPolicies(subscriptionEnabledAIRatelimitPolicies map[apimachiner_types.NamespacedName]struct{}, aiRatelimitPolicySpecs map[apimachiner_types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + rlsPolicyCache.ProcessSubscriptionBasedAIRatelimitPolicySpecsAndUpdateCache(subscriptionEnabledAIRatelimitPolicies, aiRatelimitPolicySpecs) +} + // DeleteAPILevelRateLimitPolicies delete the ratelimit xds cache func DeleteAPILevelRateLimitPolicies(resolveRatelimitPolicyList []dpv1alpha1.ResolveRateLimitAPIPolicy) { From e4db1af13e69d41fa11a821cafcab1fe29fd0d28 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Tue, 10 Sep 2024 20:35:50 +0530 Subject: [PATCH 05/24] Add sub based ai ratelimit feature related changes --- .../oasparser/envoyconf/routes_configs.go | 5 +++- .../envoyconf/routes_with_clusters.go | 1 + .../operator/controllers/dp/api_controller.go | 14 ++++++--- adapter/internal/operator/operator.go | 2 +- .../operator/synchronizer/data_store.go | 5 ++++ .../internal/xds/ratelimiter_cache.go | 3 +- .../apk/enforcer/grpc/ExtAuthService.java | 1 + .../grpc/ExternalProcessorService.java | 29 ++++++++++++++++--- .../enforcer/grpc/client/RatelimitClient.java | 24 ++++++++++++--- .../security/jwt/Oauth2Authenticator.java | 2 ++ 10 files changed, 71 insertions(+), 15 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index 8913a1e5d0..4798a48cd5 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -152,6 +152,9 @@ func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, rate if isBackendBasedAIRatelimitEnabled { action.Route.RateLimits = append(action.Route.RateLimits, generateBackendBasedAIRatelimit(descriptorValueForBackendBasedAIRatelimit)...) } + if isSubscriptionBasedAIRatelimitEnabled { + action.Route.RateLimits = append(action.Route.RateLimits, generateSubscriptionBasedAIRatelimit()...) + } // Add request mirroring configurations if mirrorClusterNames != nil && len(mirrorClusterNames) > 0 { @@ -260,7 +263,7 @@ func generateBackendBasedAIRatelimit(descValue string) []*routev3.RateLimit { } -func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { +func generateSubscriptionBasedAIRatelimit() []*routev3.RateLimit { rateLimitForRequestTokenCount := routev3.RateLimit{ Actions: []*routev3.RateLimit_Action{ { diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 674f1c8086..4d0928bb7c 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -930,6 +930,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } routeConfig := resource.GetEndpoints().Config metaData := &corev3.Metadata{} + logger.LoggerAPI.Infof("Is backend based rl enabled: %+v, Is subs based rl enabled: %+v", resource.GetEnableBackendBasedAIRatelimit(), resource.GetEnableSubscriptionBasedAIRatelimit()) if resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit() { metaData = &corev3.Metadata{ FilterMetadata: map[string]*structpb.Struct{ diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 09c3bd1c4a..309a85c21a 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -865,12 +865,18 @@ func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx c } for _, subscription := range subscriptionList.Items { aiRatelimitPolicy := &dpv1alpha3.AIRateLimitPolicy{} - if err := apiReconciler.client.Get(ctx, utils.NamespacedName(&subscription), aiRatelimitPolicy, ); err != nil { + nn:= types.NamespacedName{ + Name: subscription.Spec.RatelimitRef.Name, + Namespace: subscription.GetNamespace(), + } + if err := apiReconciler.client.Get(ctx, nn, aiRatelimitPolicy, ); err != nil { loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) - return + continue + } else { + loggers.LoggerAPKOperator.Infof("API state set as AI subscription enabled") + apiState.IsAiSubscriptionRatelimitEnabled = true + break } - apiState.IsAiSubscriptionRatelimitEnabled = true - return } } diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index b7236ce5ed..ece07386a8 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -51,7 +51,7 @@ import ( dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" - cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" + cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3" //+kubebuilder:scaffold:imports ) diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 1ecb67f1c2..23183f9491 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -75,6 +75,11 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced var updated bool events := []string{} cachedAPI := ods.apiStore[apiNamespacedName] + if cachedAPI.IsAiSubscriptionRatelimitEnabled != apiState.IsAiSubscriptionRatelimitEnabled { + cachedAPI.IsAiSubscriptionRatelimitEnabled = apiState.IsAiSubscriptionRatelimitEnabled + updated = true + events = append(events, "Subscription based AI RatelimitPolicy") + } if apiState.APIDefinition.Generation > cachedAPI.APIDefinition.Generation { cachedAPI.APIDefinition = apiState.APIDefinition diff --git a/common-controller/internal/xds/ratelimiter_cache.go b/common-controller/internal/xds/ratelimiter_cache.go index 7f7d918a10..53827e7230 100644 --- a/common-controller/internal/xds/ratelimiter_cache.go +++ b/common-controller/internal/xds/ratelimiter_cache.go @@ -448,7 +448,8 @@ func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRat } func prepareSubscriptionBasedAIRatelimitIdentifier(org string, namespacedName types.NamespacedName) string { - return fmt.Sprintf("%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name)) + // return fmt.Sprintf("%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name)) + return fmt.Sprintf("%s-%s", org, string(namespacedName.Name)) } func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index eb2b59a978..267a0b7305 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -264,6 +264,7 @@ private String constructQueryParamString(boolean removeAllQueryParams, String re * @param value */ private void addMetadata(Struct.Builder structBuilder, String key, String value) { + System.out.println("Key: " + key + " value: " + value); structBuilder.putFields(key, Value.newBuilder().setStringValue(value).build()); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index bda06b8e84..1678eea94c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -18,10 +18,13 @@ package org.wso2.apk.enforcer.grpc; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.Struct; import com.google.protobuf.Value; +import io.envoyproxy.envoy.config.core.v3.Metadata; import io.envoyproxy.envoy.service.ext_proc.v3.BodyMutation; import io.envoyproxy.envoy.service.ext_proc.v3.BodyResponse; import io.envoyproxy.envoy.service.ext_proc.v3.CommonResponse; @@ -47,6 +50,12 @@ public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProc private static final String DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT = "airequesttokencount"; private static final String DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT = "airesponsetokencount"; private static final String DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT = "aitotaltokencount"; + private static final String DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT = "airequesttokencountsubs"; + private static final String DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT = "airesponsetokencountsubs"; + private static final String DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT = "aitotaltokencountsubs"; + private static final String DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION = "subscription"; + private static final String DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY = "ratelimit:organization-and-rlpolicy"; + private static final String DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION = "ratelimit:subscription"; RatelimitClient ratelimitClient = new RatelimitClient(); @Override public StreamObserver process( @@ -92,13 +101,25 @@ public void onNext(ProcessingRequest request) { return; } System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); + List configs = new ArrayList<>(); if (filterMetadata.enableBackendBasedAIRatelimit) { - List configs = new ArrayList<>(); configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); - ratelimitClient.shouldRatelimit(configs); } + if (filterMetadata.enableSubscriptionBasedAIRatelimit) { + if (request.hasMetadataContext()) { + Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); + if (filterMetadataFromAuthZ != null) { + String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); + String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + } + } + } + ratelimitClient.shouldRatelimit(configs); responseObserver.onCompleted(); } else { System.out.println("Request does not have response body"); @@ -258,8 +279,8 @@ private static Usage extractUsageFromBody(String body, String completionTokenPat System.out.println("Usage extracted: "+ usage); return usage; - } catch (JsonProcessingException e) { - logger.error(String.format("Unexpected error while extracting usage from the body: %s", body), e); + } catch (Exception e) { + System.out.println(String.format("Unexpected error while extracting usage from the body: %s", body) + " \n" + e); return null; } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java index 3034296060..fb528ffbd0 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -52,14 +52,22 @@ public void shouldRatelimit(List configs) { executorService.submit(() -> { System.out.println("Ratelimitclient test"); for (KeyValueHitsAddend config : configs) { + System.out.println("For: " + config.getKey()); - RateLimitDescriptor descriptor = RateLimitDescriptor.newBuilder() - .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()) - .build(); + RateLimitDescriptor.Builder builder = RateLimitDescriptor.newBuilder() + .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()); + KeyValueHitsAddend internalKeyValueHitsAddend = config.keyValueHitsAddend; + int hitsAddend = config.getHitsAddend(); + while (internalKeyValueHitsAddend != null) { + builder.addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(internalKeyValueHitsAddend.getKey()).setValue(internalKeyValueHitsAddend.getValue()).build()); + hitsAddend = internalKeyValueHitsAddend.getHitsAddend(); + internalKeyValueHitsAddend = internalKeyValueHitsAddend.keyValueHitsAddend; + } + RateLimitDescriptor descriptor = builder.build(); RateLimitRequest rateLimitRequest = RateLimitRequest.newBuilder() .addDescriptors(descriptor) .setDomain("Default") - .setHitsAddend(config.getHitsAddend()) + .setHitsAddend(hitsAddend) .build(); RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); System.out.println(rateLimitResponse.getOverallCode()); @@ -73,11 +81,19 @@ public static class KeyValueHitsAddend { private String key; private String value; private int hitsAddend; + private KeyValueHitsAddend keyValueHitsAddend; public KeyValueHitsAddend(String key, String value, int hitsAddend) { this.key = key; this.value = value; this.hitsAddend = hitsAddend; + this.keyValueHitsAddend = null; + } + public KeyValueHitsAddend(String key, String value, KeyValueHitsAddend keyValueHitsAddend) { + this.key = key; + this.value = value; + this.hitsAddend = -1; + this.keyValueHitsAddend = keyValueHitsAddend; } public String getKey() { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java index 9a29fbf451..b12dda1c0a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java @@ -268,6 +268,8 @@ public AuthenticationContext authenticate(RequestContext requestContext) throws requestContext.addMetadataToMap("ratelimit:subscription", subscriptionId); requestContext.addMetadataToMap("ratelimit:usage-policy", subscription.getRatelimitTier()); requestContext.addMetadataToMap("ratelimit:organization", subscription.getOrganization()); + System.out.println("Value: "+ String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + requestContext.addMetadataToMap("ratelimit:organization-and-rlpolicy", String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); } } } From 27181bb38a56b71090aa6b22dba2ace044aef9ca Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 11 Sep 2024 11:41:50 +0530 Subject: [PATCH 06/24] Fix intermittent ratelimit timeout in envoy --- adapter/internal/oasparser/envoyconf/http_filters.go | 4 ++++ .../org/wso2/apk/enforcer/grpc/ExternalProcessorService.java | 4 ++-- .../org/wso2/apk/enforcer/grpc/client/RatelimitClient.java | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/http_filters.go b/adapter/internal/oasparser/envoyconf/http_filters.go index 06867d6c76..4017426f87 100644 --- a/adapter/internal/oasparser/envoyconf/http_filters.go +++ b/adapter/internal/oasparser/envoyconf/http_filters.go @@ -169,6 +169,10 @@ func getRateLimitFilter() *hcmv3.HttpFilter { Domain: RateLimiterDomain, FailureModeDeny: conf.Envoy.RateLimit.FailureModeDeny, EnableXRatelimitHeaders: enableXRatelimitHeaders, + Timeout: &durationpb.Duration{ + Nanos: (int32(conf.Envoy.RateLimit.RequestTimeoutInMillis) % 1000) * 1000000, + Seconds: conf.Envoy.RateLimit.RequestTimeoutInMillis / 1000, + }, RateLimitService: &envoy_config_ratelimit_v3.RateLimitServiceConfig{ TransportApiVersion: corev3.ApiVersion_V3, GrpcService: &corev3.GrpcService{ diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index 1678eea94c..402fa76e00 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -114,8 +114,8 @@ public void onNext(ProcessingRequest request) { String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); } } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java index fb528ffbd0..590a492f03 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -70,6 +70,7 @@ public void shouldRatelimit(List configs) { .setHitsAddend(hitsAddend) .build(); RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); + System.out.println("ratelimit response: " + rateLimitResponse.getStatuses(0).getCurrentLimit() + " " + rateLimitResponse.getStatuses(0).getLimitRemaining()); System.out.println(rateLimitResponse.getOverallCode()); } }); From 7f9eed62b3d4bb8514a5336957dbf76bdc02b0e2 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 11 Sep 2024 15:12:44 +0530 Subject: [PATCH 07/24] Use AIProvider to extract token details from response --- .../oasparser/envoyconf/internal_dtos.go | 1 + .../envoyconf/routes_with_clusters.go | 23 ++++--- .../operator/controllers/dp/api_controller.go | 6 +- .../operator/synchronizer/data_store.go | 14 ++++ .../grpc/ExternalProcessorService.java | 67 ++++++++++++------- .../apk/enforcer/security/AuthFilter.java | 6 ++ 6 files changed, 77 insertions(+), 40 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/internal_dtos.go b/adapter/internal/oasparser/envoyconf/internal_dtos.go index 1ca8d5f212..0cb5f6b2b2 100644 --- a/adapter/internal/oasparser/envoyconf/internal_dtos.go +++ b/adapter/internal/oasparser/envoyconf/internal_dtos.go @@ -45,6 +45,7 @@ type routeCreateParams struct { environment string envType string mirrorClusterNames map[string][]string + isAiAPI bool } // RatelimitCriteria criterias of rate limiting diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 4d0928bb7c..24abc8ceb9 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -865,7 +865,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error LuaLocal: luaFilter, wellknown.CORS: corsFilter, } - if !resource.GetEnableBackendBasedAIRatelimit() && !resource.GetEnableSubscriptionBasedAIRatelimit() { + if !params.isAiAPI || (!resource.GetEnableBackendBasedAIRatelimit() && !resource.GetEnableSubscriptionBasedAIRatelimit()) { perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ Override: &extProcessorv3.ExtProcPerRoute_Disabled{ Disabled: true, @@ -878,7 +878,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } perRouteFilterConfigs[HTTPExternalProcessor] = filterExtProc } - + logger.LoggerOasparser.Debugf("adding route : %s for API : %s", resourcePath, title) rateLimitPolicyLevel := "" @@ -931,7 +931,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error routeConfig := resource.GetEndpoints().Config metaData := &corev3.Metadata{} logger.LoggerAPI.Infof("Is backend based rl enabled: %+v, Is subs based rl enabled: %+v", resource.GetEnableBackendBasedAIRatelimit(), resource.GetEnableSubscriptionBasedAIRatelimit()) - if resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit() { + if params.isAiAPI && (resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit()) { metaData = &corev3.Metadata{ FilterMetadata: map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": &structpb.Struct{ @@ -1092,8 +1092,8 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error metadataValue := operation.GetMethod() + "_to_" + newMethod match2.DynamicMetadata = generateMetadataMatcherForInternalRoutes(metadataValue) - action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) - action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) + action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) + action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) // Create route1 for current method. // Do not add policies to route config. Send via enforcer @@ -1116,7 +1116,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else { var action *routev3.Route_Route if requestRedirectAction == nil { - action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) + action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) } logger.LoggerOasparser.Debug("Creating routes for resource with policies", resourcePath, operation.GetMethod()) // create route for current method. Add policies to route config. Send via enforcer @@ -1145,7 +1145,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) - + route := generateRouteConfig(xWso2Basepath, match, action, nil, metaData, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) @@ -1284,7 +1284,7 @@ func CreateAPIDefinitionRoute(basePath string, vHost string, methods []string, i Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, - HTTPExternalProcessor : filterExtProc, + HTTPExternalProcessor: filterExtProc, }, } return &router @@ -1378,7 +1378,7 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, - HTTPExternalProcessor : filterExtProc, + HTTPExternalProcessor: filterExtProc, }, } return router @@ -1443,7 +1443,7 @@ func CreateHealthEndpoint() *routev3.Route { Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, - HTTPExternalProcessor : filterExtProc, + HTTPExternalProcessor: filterExtProc, }, } return &router @@ -1493,7 +1493,7 @@ func CreateReadyEndpoint() *routev3.Route { Metadata: nil, Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ - HTTPExternalProcessor : filterExtProc, + HTTPExternalProcessor: filterExtProc, }, } return &router @@ -1691,6 +1691,7 @@ func genRouteCreateParams(swagger *model.AdapterInternalAPI, resource *model.Res environment: swagger.GetEnvironment(), envType: swagger.EnvType, mirrorClusterNames: mirrorClusterNames, + isAiAPI: swagger.AIProvider.Enabled, } return params } diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 309a85c21a..4ccc105df3 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -869,14 +869,12 @@ func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx c Name: subscription.Spec.RatelimitRef.Name, Namespace: subscription.GetNamespace(), } - if err := apiReconciler.client.Get(ctx, nn, aiRatelimitPolicy, ); err != nil { - loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) - continue - } else { + if err := apiReconciler.client.Get(ctx, nn, aiRatelimitPolicy, ); err == nil { loggers.LoggerAPKOperator.Infof("API state set as AI subscription enabled") apiState.IsAiSubscriptionRatelimitEnabled = true break } + loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) } } diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 23183f9491..8d8f4bb50b 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -81,6 +81,20 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced events = append(events, "Subscription based AI RatelimitPolicy") } + if cachedAPI.AIProvider == nil && apiState.AIProvider != nil { + cachedAPI.AIProvider = apiState.AIProvider + updated = true + events = append(events, "API provider") + } else if cachedAPI.AIProvider != nil && apiState.AIProvider == nil{ + cachedAPI.AIProvider = nil + updated = true + events = append(events, "API provider") + } else if cachedAPI.AIProvider.Generation != apiState.AIProvider.Generation { + cachedAPI.AIProvider = apiState.AIProvider + updated = true + events = append(events, "API provider") + } + if apiState.APIDefinition.Generation > cachedAPI.APIDefinition.Generation { cachedAPI.APIDefinition = apiState.APIDefinition updated = true diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index 402fa76e00..ade43dfa05 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -56,6 +56,10 @@ public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProc private static final String DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION = "subscription"; private static final String DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY = "ratelimit:organization-and-rlpolicy"; private static final String DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION = "ratelimit:subscription"; + private static final String DYNAMIC_METADATA_KEY_FOR_EXTRACT_TOKEN_FROM = "aitoken:extracttokenfrom"; + private static final String DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID = "aitoken:prompttokenid"; + private static final String DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID = "aitoken:completiontokenid"; + private static final String DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID = "aitoken:totaltokenid"; RatelimitClient ratelimitClient = new RatelimitClient(); @Override public StreamObserver process( @@ -92,34 +96,47 @@ public void onNext(ProcessingRequest request) { System.out.println("In the response flow metadata descirtor:" + filterMetadata.backendBasedAIRatelimitDescriptorValue); if (request.hasResponseBody()) { String body = request.getResponseBody().getBody().toStringUtf8(); -// System.out.println("Body: " + body); - Usage usage = extractUsageFromBody(body, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); - if (usage == null) { - logger.error("Usage details not found.."); - System.out.println("Usage details not found.."); - responseObserver.onCompleted(); - return; - } - System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); - List configs = new ArrayList<>(); - if (filterMetadata.enableBackendBasedAIRatelimit) { - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); - } - if (filterMetadata.enableSubscriptionBasedAIRatelimit) { - if (request.hasMetadataContext()) { - Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); - if (filterMetadataFromAuthZ != null) { - String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); - String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); + Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); + if (filterMetadataFromAuthZ != null) { + String extractTokenFrom = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_EXTRACT_TOKEN_FROM).getStringValue(); + System.out.println("Extract Token From: " + extractTokenFrom); + + String promptTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID).getStringValue(); + System.out.println("Prompt Token ID: " + promptTokenID); + + String completionTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID).getStringValue(); + System.out.println("Completion Token ID: " + completionTokenID); + + String totalTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID).getStringValue(); + System.out.println("Total Token ID: " + totalTokenID); + + Usage usage = extractUsageFromBody(body, completionTokenID, promptTokenID, totalTokenID); + if (usage == null) { + logger.error("Usage details not found.."); + System.out.println("Usage details not found.."); + responseObserver.onCompleted(); + return; + } + System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); + List configs = new ArrayList<>(); + if (filterMetadata.enableBackendBasedAIRatelimit) { + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + } + if (filterMetadata.enableSubscriptionBasedAIRatelimit) { + if (request.hasMetadataContext()) { + if (filterMetadataFromAuthZ != null) { + String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); + String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); + } } } + ratelimitClient.shouldRatelimit(configs); } - ratelimitClient.shouldRatelimit(configs); responseObserver.onCompleted(); } else { System.out.println("Request does not have response body"); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java index e3e2fed115..6b2d2a855f 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java @@ -137,6 +137,12 @@ public boolean handleRequest(RequestContext requestContext) { boolean authenticated = false; // Any auth token has been provided for application-level security or not boolean canAuthenticated = false; + if (requestContext.getMatchedAPI() != null && requestContext.getMatchedAPI().getAiProvider() != null) { + requestContext.addMetadataToMap("aitoken:prompttokenid", requestContext.getMatchedAPI().getAiProvider().getPromptTokens().getValue()); + requestContext.addMetadataToMap("aitoken:completiontokenid", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getValue()); + requestContext.addMetadataToMap("aitoken:totaltokenid", requestContext.getMatchedAPI().getAiProvider().getTotalToken().getValue()); + requestContext.addMetadataToMap("aitoken:extracttokenfrom", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getIn()); + } for (Authenticator authenticator : authenticators) { if (authenticator.canAuthenticate(requestContext)) { // For transport level securities (mTLS), canAuthenticated will not be applied From 4f0ad81c4be9b48e1d3dd908f6f53e32d6ae3c46 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 11 Sep 2024 15:21:50 +0530 Subject: [PATCH 08/24] Fix revive errors --- common-controller/internal/cache/datastore.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common-controller/internal/cache/datastore.go b/common-controller/internal/cache/datastore.go index b21ba84b74..eb3b00ff16 100644 --- a/common-controller/internal/cache/datastore.go +++ b/common-controller/internal/cache/datastore.go @@ -103,7 +103,7 @@ func (ods *RatelimitDataStore) AddorUpdateAIRatelimitToStore(rateLimit types.Nam ods.aiRatelimitPolicySpecs[rateLimit] = &aiRatelimitSpec } -// MarkAIRatelimitAsSubscriptionEnabled +// MarkAIRatelimitAsSubscriptionEnabled add an entry to specify an AI RatelimitPolicy is associated with a subscription func (ods *RatelimitDataStore) MarkAIRatelimitAsSubscriptionEnabled(nn types.NamespacedName) { ods.mu.Lock() defer ods.mu.Unlock() @@ -113,7 +113,7 @@ func (ods *RatelimitDataStore) MarkAIRatelimitAsSubscriptionEnabled(nn types.Nam ods.subscriptionEnabledAIRatelimitPolicies[nn] = struct{}{} } -// MarkAIRatelimitAsSubscriptionDisabled +// MarkAIRatelimitAsSubscriptionDisabled deletes the entry which was added to specify an AI RatelimitPolicy is associated with a subscription func (ods *RatelimitDataStore) MarkAIRatelimitAsSubscriptionDisabled(nn types.NamespacedName) { ods.mu.Lock() defer ods.mu.Unlock() @@ -148,7 +148,7 @@ func (ods *RatelimitDataStore) GetAIRatelimitPolicySpecs() map[types.NamespacedN return ods.aiRatelimitPolicySpecs } -// GetAIRatelimitPolicySpecs gets all the AIRatelimitPolicy stored in ods +// GetSubscriptionEnabledAIRatelimitPolicies gets all the AIRatelimitPolicy stored in ods func (ods *RatelimitDataStore) GetSubscriptionEnabledAIRatelimitPolicies() map[types.NamespacedName]struct{} { return ods.subscriptionEnabledAIRatelimitPolicies } From 3333443e58b7956d1966daf10ce28b43d1b21a98 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 11 Sep 2024 16:00:56 +0530 Subject: [PATCH 09/24] Fix ratelimittier not getting from restapi problem --- .../subscription/SubscriptionDataStoreImpl.java | 1 + .../apk/enforcer/subscription/SubscriptionDto.java | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java index 4859bac170..5f8a3ccad4 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDataStoreImpl.java @@ -99,6 +99,7 @@ public void addSubscriptions(List subscriptionList) { newSubscription.setOrganization(subscription.getOrganization()); newSubscription.setSubscribedApi(subscribedAPI); newSubscriptionMap.put(newSubscription.getCacheKey(), newSubscription); + newSubscription.setRatelimitTier(subscription.getRatelimitTier()); } if (log.isDebugEnabled()) { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java index c2e7133c1b..b8a31efaad 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java @@ -12,6 +12,7 @@ public class SubscriptionDto implements Serializable { private String organization; private String subStatus; private SubscribedAPIDto subscribedApi; + private String ratelimitTier; public String getUuid() { @@ -52,4 +53,14 @@ public void setSubscribedApi(SubscribedAPIDto subscribedApi) { this.subscribedApi = subscribedApi; } + + public String getRatelimitTier() { + + return ratelimitTier; + } + + public void setRatelimitTier(String ratelimitTier) { + + this.ratelimitTier = ratelimitTier; + } } \ No newline at end of file From 4adc4e19ac0b9fba124a92e8447ba1926fd2bafb Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 11 Sep 2024 16:12:39 +0530 Subject: [PATCH 10/24] Fix NPE --- .../org/wso2/apk/enforcer/security/AuthFilter.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java index 6b2d2a855f..250b56c930 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/AuthFilter.java @@ -138,10 +138,16 @@ public boolean handleRequest(RequestContext requestContext) { // Any auth token has been provided for application-level security or not boolean canAuthenticated = false; if (requestContext.getMatchedAPI() != null && requestContext.getMatchedAPI().getAiProvider() != null) { - requestContext.addMetadataToMap("aitoken:prompttokenid", requestContext.getMatchedAPI().getAiProvider().getPromptTokens().getValue()); - requestContext.addMetadataToMap("aitoken:completiontokenid", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getValue()); - requestContext.addMetadataToMap("aitoken:totaltokenid", requestContext.getMatchedAPI().getAiProvider().getTotalToken().getValue()); - requestContext.addMetadataToMap("aitoken:extracttokenfrom", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getIn()); + if (requestContext.getMatchedAPI().getAiProvider().getPromptTokens() != null) { + requestContext.addMetadataToMap("aitoken:prompttokenid", requestContext.getMatchedAPI().getAiProvider().getPromptTokens().getValue()); + } + if (requestContext.getMatchedAPI().getAiProvider().getCompletionToken() != null) { + requestContext.addMetadataToMap("aitoken:extracttokenfrom", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getIn()); + requestContext.addMetadataToMap("aitoken:completiontokenid", requestContext.getMatchedAPI().getAiProvider().getCompletionToken().getValue()); + } + if (requestContext.getMatchedAPI().getAiProvider().getTotalToken() != null) { + requestContext.addMetadataToMap("aitoken:totaltokenid", requestContext.getMatchedAPI().getAiProvider().getTotalToken().getValue()); + } } for (Authenticator authenticator : authenticators) { if (authenticator.canAuthenticate(requestContext)) { From 1987cee2e67110f6f61a8f06fa4fc20625793d4d Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Thu, 12 Sep 2024 19:05:41 +0530 Subject: [PATCH 11/24] Everything works --- .../envoyconf/routes_with_clusters.go | 23 ++- .../apk/enforcer/grpc/ExtAuthService.java | 8 + .../grpc/ExternalProcessorService.java | 151 ++++++++++-------- .../enforcer/grpc/client/RatelimitClient.java | 53 +++--- .../apk/enforcer/security/KeyValidator.java | 2 + .../security/jwt/APIKeyAuthenticator.java | 58 ++++++- libs.versions.toml | 2 + 7 files changed, 191 insertions(+), 106 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 24abc8ceb9..a4fb91b1c8 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -1095,10 +1095,16 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) + requestHeadersToRemove := make([]string,0) + if params.isAiAPI { + requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") + } else { + requestHeadersToRemove = nil + } // Create route1 for current method. // Do not add policies to route config. Send via enforcer route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, - nil, nil, nil, nil) + nil, requestHeadersToRemove, nil, nil) // Create route2 for new method. // Add all policies to route config. Do not send via enforcer. @@ -1108,6 +1114,9 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action2.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } configToSkipEnforcer := generateFilterConfigToSkipEnforcer() + if params.isAiAPI { + requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") + } route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, metaData, decorator, configToSkipEnforcer, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) @@ -1128,6 +1137,9 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else if requestRedirectAction == nil { action.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } + if params.isAiAPI { + requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") + } route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route) @@ -1145,9 +1157,14 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) - + requestHeadersToRemove := make([]string,0) + if params.isAiAPI { + requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") + } else { + requestHeadersToRemove = nil + } route := generateRouteConfig(xWso2Basepath, match, action, nil, metaData, decorator, perRouteFilterConfigs, - nil, nil, nil, nil) // general headers to add and remove are included in this methods + nil, requestHeadersToRemove, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) } return routes, nil diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index 267a0b7305..b961743b37 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -52,9 +52,15 @@ import org.wso2.apk.enforcer.tracing.TracingTracer; import org.wso2.apk.enforcer.tracing.Utils; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; /** * This is the gRPC server written to match with the envoy ext-authz filter proto file. Envoy proxy call this service. @@ -303,4 +309,6 @@ private CheckResponse buildReadyCheckResponse(CheckResponse.Builder responseBuil .setDeniedResponse(deniedResponsePreparer.build()) .build(); } + + } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index ade43dfa05..7524c30f35 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -18,26 +18,33 @@ package org.wso2.apk.enforcer.grpc; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.Struct; import com.google.protobuf.Value; -import io.envoyproxy.envoy.config.core.v3.Metadata; import io.envoyproxy.envoy.service.ext_proc.v3.BodyMutation; import io.envoyproxy.envoy.service.ext_proc.v3.BodyResponse; import io.envoyproxy.envoy.service.ext_proc.v3.CommonResponse; import io.envoyproxy.envoy.service.ext_proc.v3.ExternalProcessorGrpc; +import io.envoyproxy.envoy.service.ext_proc.v3.HeaderMutation; import io.envoyproxy.envoy.service.ext_proc.v3.HeadersResponse; import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingRequest; import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingResponse; import io.grpc.stub.StreamObserver; +import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.wso2.apk.enforcer.grpc.client.RatelimitClient; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -60,6 +67,7 @@ public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProc private static final String DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID = "aitoken:prompttokenid"; private static final String DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID = "aitoken:completiontokenid"; private static final String DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID = "aitoken:totaltokenid"; + private final ExecutorService executorService = Executors.newFixedThreadPool(10);; RatelimitClient ratelimitClient = new RatelimitClient(); @Override public StreamObserver process( @@ -70,73 +78,62 @@ public StreamObserver process( @Override public void onNext(ProcessingRequest request) { - System.out.println("on next ...."); ProcessingRequest.RequestCase r = request.getRequestCase(); - System.out.println("case: " + r.name()); switch (r) { - case REQUEST_HEADERS: - if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ - Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); - FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); - System.out.println("Metadata generated: "+ metadata); - filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; - filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; - filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; - } - responseObserver.onNext(ProcessingResponse.newBuilder().build()); case RESPONSE_BODY: if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); - System.out.println("Metadata generated: "+ metadata); filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; } System.out.println("In the response flow metadata descirtor:" + filterMetadata.backendBasedAIRatelimitDescriptorValue); if (request.hasResponseBody()) { - String body = request.getResponseBody().getBody().toStringUtf8(); - Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); - if (filterMetadataFromAuthZ != null) { - String extractTokenFrom = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_EXTRACT_TOKEN_FROM).getStringValue(); - System.out.println("Extract Token From: " + extractTokenFrom); - - String promptTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID).getStringValue(); - System.out.println("Prompt Token ID: " + promptTokenID); - - String completionTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID).getStringValue(); - System.out.println("Completion Token ID: " + completionTokenID); - - String totalTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID).getStringValue(); - System.out.println("Total Token ID: " + totalTokenID); - - Usage usage = extractUsageFromBody(body, completionTokenID, promptTokenID, totalTokenID); - if (usage == null) { - logger.error("Usage details not found.."); - System.out.println("Usage details not found.."); - responseObserver.onCompleted(); - return; - } - System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); - List configs = new ArrayList<>(); - if (filterMetadata.enableBackendBasedAIRatelimit) { - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + final byte[] bodyFromResponse = request.getResponseBody().getBody().toByteArray(); + executorService.submit(() -> { + String body = null; + try { + body = decompress(bodyFromResponse); + } catch (Exception e) { + throw new RuntimeException(e); } - if (filterMetadata.enableSubscriptionBasedAIRatelimit) { - if (request.hasMetadataContext()) { - if (filterMetadataFromAuthZ != null) { - String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); - String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); + + Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); + if (filterMetadataFromAuthZ != null) { + String extractTokenFrom = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_EXTRACT_TOKEN_FROM).getStringValue(); + String promptTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID).getStringValue(); + String completionTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID).getStringValue(); + String totalTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID).getStringValue(); + + Usage usage = extractUsageFromBody(body, completionTokenID, promptTokenID, totalTokenID); + if (usage == null) { + logger.error("Usage details not found.."); + responseObserver.onCompleted(); + return; + } + List configs = new ArrayList<>(); + if (filterMetadata.enableBackendBasedAIRatelimit) { + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + } + if (filterMetadata.enableSubscriptionBasedAIRatelimit) { + if (request.hasMetadataContext()) { + if (filterMetadataFromAuthZ != null) { + if (filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY) != null && filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION) != null) { + String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); + String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); + } + } } } + ratelimitClient.shouldRatelimit(configs); } - ratelimitClient.shouldRatelimit(configs); - } + }); responseObserver.onCompleted(); } else { System.out.println("Request does not have response body"); @@ -153,7 +150,6 @@ public void onError(Throwable err) { @Override public void onCompleted() { - System.out.println("on completed ...."); responseObserver.onCompleted(); } }; @@ -174,6 +170,9 @@ protected HeadersResponse prepareHeadersResponse() { .setResponse( CommonResponse.newBuilder() .setStatus(CommonResponse.ResponseStatus.CONTINUE) + .setHeaderMutation( + HeaderMutation.newBuilder() + .build()) .setBodyMutation(BodyMutation.newBuilder().build()) .build()) .build(); @@ -197,7 +196,6 @@ public String toString() { // Method to parse the string and create FilterMetadata object public static FilterMetadata convertStringToFilterMetadata(String input) { FilterMetadata metadata = new FilterMetadata(); - // Regex patterns to extract specific fields String backendValuePattern = "key: \"BackendBasedAIRatelimitDescriptorValue\".*?string_value: \"(.*?)\""; String enableBackendPattern = "key: \"EnableBackendBasedAIRatelimit\".*?string_value: \"(.*?)\""; @@ -221,16 +219,6 @@ private static String extractValue(String input, String pattern) { return null; } - - public static void main(String[] args) { - String input = "filter_metadata { key: \"envoy.filters.http.ext_proc\" value { fields { key: \"BackendBasedAIRatelimitDescriptorValue\" value { string_value: \"default-apk-backend-ratelimit-xxx-apk-backend-93fa16a00dbec9ff438aa40e6358d91dcdc22f48-api\" } } fields { key: \"EnableBackendBasedAIRatelimit\" value { string_value: \"true\" } } fields { key: \"EnableSubscriptionBasedAIRatelimit\" value { string_value: \"false\" } } } }"; - input = "{ \"choices\":[ { \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } }, \"finish_reason\":\"stop\", \"index\":0, \"logprobs\":null, \"message\":{ \"content\":\"Arr, matey! Ye be askin' a great question. Care for a parrot be a task that requires careful attention and love. Here be some tips to keep yer feathered friend happy and healthy: 1. Provide a proper cage: A parrot needs an adequate-sized cage to freely stretch its wings. Make sure it has enough space for perching, playing, and spreading those beautiful feathers. Also, ensure the bars are close enough together to prevent escape. 2. Nourishing grub: A parrot's diet be important. Offer a balanced diet of high-quality parrot pellets, fresh fruits, vegetables, and some seeds. Avoid avacados, chocolate, caffeine, and anythin' toxic to a bird's delicate system. 3. Fresh water: Change the water in yer parrot's bowl daily, matey. Keeps it clean and fresh. Parrots love to dunk their beaks, so ensure they have ample water for sippin' and splish-splashin'. 4. Feathered entertainment: Parrots be social creatures and need mental stimulation. Provide 'em with plenty of toys, such as ropes, bells, and puzzle toys, to keep 'em entertained. Rotate the toys frequently to avoid boredom. 5. Avast, matey! Give 'em attention: Parrots be fond of interaction with their human companions. Spend time talkin' to 'em, singin' shanties, and makin' 'em feel loved. They may even learn a few words or phrases! 6. Exercise be important: Encourage yer parrot to exercise its wings, me hearty. Free-flyin' in a safe, enclosed area be ideal, or let 'em out of their cage for supervised playtime. 7. Regular vet visits: Aye, take yer parrot to a qualified avian vet for regular check-ups. They'll make sure yer feathered friend's health be in shipshape and suggest any necessary vaccinations or treatments. 8. Aye, watch for signs of illness: Keep a keen eye on yer parrot for any signs of illness, such as changes in appetite, behavior, or feather condition. If ye spot any concerns, consult an avian vet right quick. Remember, matey, each parrot be unique, so get to know yer bird and pay attention to its specific needs. Aye, with proper care, ye and yer parrot will forge a bond that be stronger than the mightiest of pirate ships. Fair winds and happy parrot keepin'!\", \"role\":\"assistant\" } } ], \"created\":1724232516, \"id\":\"chatcmpl-9ybvo8dQte9Hb0IkD2NCaOME9Q1LH\", \"model\":\"gpt-35-turbo\", \"object\":\"chat.completion\", \"prompt_filter_results\":[ { \"prompt_index\":0, \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } } } ], \"system_fingerprint\":null, \"usage\":{ \"completion_tokens\":514, \"prompt_tokens\":33, \"total_tokens\":547 } }"; - Usage usage = extractUsageFromBody(input, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); - System.out.println(usage.completion_tokens); -// FilterMetadata metadata = convertStringToFilterMetadata(input); -// System.out.println(metadata.backendBasedAIRatelimitDescriptorValue); // Printing the FilterMetadata object - } - public static String sanitize(String input) { // Replace all newline characters and tabs with a space return input.replaceAll("[\\t\\n\\r]+", " ").trim(); @@ -293,7 +281,6 @@ private static Usage extractUsageFromBody(String body, String completionTokenPat } } usage.setTotal_tokens(currentNodeForTotalToken.asInt()); - System.out.println("Usage extracted: "+ usage); return usage; } catch (Exception e) { @@ -338,4 +325,34 @@ public String toString() { } } + public static String decompress(byte[] compressed) throws Exception { + String body = new String(compressed, StandardCharsets.UTF_8); + if (isValidJson(body)) { + return body; + } + try (InputStream is = new CompressorStreamFactory() + .createCompressorInputStream(new ByteArrayInputStream(compressed)); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + StringBuilder outStr = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + outStr.append(line); + } + if (isValidJson(outStr.toString())) { + return outStr.toString(); + } else { + throw new RuntimeException("Could not decompress response body"); + } + } + } + + public static boolean isValidJson(String json) { + try { + ObjectMapper mapper = new ObjectMapper(); + mapper.readTree(json); + return true; + } catch (Exception e) { + return false; + } + } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java index 590a492f03..f62d293e1c 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -19,10 +19,7 @@ public class RatelimitClient { RateLimitServiceGrpc.RateLimitServiceBlockingStub stub; - private final ExecutorService executorService; // Add an ExecutorService field - - public RatelimitClient(){//String server, int port) { - System.out.println("Ratelimitclient construct"); + public RatelimitClient(){ File certFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPublicKeyPath()).toFile(); File keyFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPrivateKeyPath()).toFile(); SslContext sslContext = null; @@ -37,45 +34,33 @@ public RatelimitClient(){//String server, int port) { } String rlHost = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterHost(); int port = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterPort(); - System.out.println("rl host and port "+ rlHost + port); ManagedChannel channel = NettyChannelBuilder.forAddress(rlHost, port) .useTransportSecurity() .sslContext(sslContext) .build(); this.stub = RateLimitServiceGrpc.newBlockingStub(channel); - // Initialize the ExecutorService - this.executorService = Executors.newFixedThreadPool(10); } public void shouldRatelimit(List configs) { -// System.out.println("RL task submitted"); - executorService.submit(() -> { - System.out.println("Ratelimitclient test"); - for (KeyValueHitsAddend config : configs) { - - System.out.println("For: " + config.getKey()); - RateLimitDescriptor.Builder builder = RateLimitDescriptor.newBuilder() - .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()); - KeyValueHitsAddend internalKeyValueHitsAddend = config.keyValueHitsAddend; - int hitsAddend = config.getHitsAddend(); - while (internalKeyValueHitsAddend != null) { - builder.addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(internalKeyValueHitsAddend.getKey()).setValue(internalKeyValueHitsAddend.getValue()).build()); - hitsAddend = internalKeyValueHitsAddend.getHitsAddend(); - internalKeyValueHitsAddend = internalKeyValueHitsAddend.keyValueHitsAddend; - } - RateLimitDescriptor descriptor = builder.build(); - RateLimitRequest rateLimitRequest = RateLimitRequest.newBuilder() - .addDescriptors(descriptor) - .setDomain("Default") - .setHitsAddend(hitsAddend) - .build(); - RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); - System.out.println("ratelimit response: " + rateLimitResponse.getStatuses(0).getCurrentLimit() + " " + rateLimitResponse.getStatuses(0).getLimitRemaining()); - System.out.println(rateLimitResponse.getOverallCode()); + for (KeyValueHitsAddend config : configs) { + RateLimitDescriptor.Builder builder = RateLimitDescriptor.newBuilder() + .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()); + KeyValueHitsAddend internalKeyValueHitsAddend = config.keyValueHitsAddend; + int hitsAddend = config.getHitsAddend(); + while (internalKeyValueHitsAddend != null) { + builder.addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(internalKeyValueHitsAddend.getKey()).setValue(internalKeyValueHitsAddend.getValue()).build()); + hitsAddend = internalKeyValueHitsAddend.getHitsAddend(); + internalKeyValueHitsAddend = internalKeyValueHitsAddend.keyValueHitsAddend; } - }); - System.out.println("RL task submitted"); - + RateLimitDescriptor descriptor = builder.build(); + RateLimitRequest rateLimitRequest = RateLimitRequest.newBuilder() + .addDescriptors(descriptor) + .setDomain("Default") + .setHitsAddend(hitsAddend) + .build(); + RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); + System.out.println(rateLimitResponse.getOverallCode()); + } } public static class KeyValueHitsAddend { diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java index c6bfc3fdae..daf2aa091a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java @@ -263,6 +263,8 @@ public static APIKeyValidationInfoDTO validateSubscription(String apiUuid, Strin return infoDTO; } + + private static void validate(APIKeyValidationInfoDTO infoDTO, Application app, Subscription sub) { // Validate subscription status diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java index 6265470bf6..24dcd43518 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java @@ -37,6 +37,7 @@ import org.wso2.apk.enforcer.commons.jwtgenerator.AbstractAPIMgtGatewayJWTGenerator; import org.wso2.apk.enforcer.commons.logging.ErrorDetails; import org.wso2.apk.enforcer.commons.logging.LoggingConstants; +import org.wso2.apk.enforcer.commons.model.APIConfig; import org.wso2.apk.enforcer.commons.model.APIKeyAuthenticationConfig; import org.wso2.apk.enforcer.commons.model.AuthenticationContext; import org.wso2.apk.enforcer.commons.model.RequestContext; @@ -48,7 +49,12 @@ import org.wso2.apk.enforcer.constants.GeneralErrorCodeConstants; import org.wso2.apk.enforcer.dto.APIKeyValidationInfoDTO; import org.wso2.apk.enforcer.dto.JWTTokenPayloadInfo; +import org.wso2.apk.enforcer.models.Application; +import org.wso2.apk.enforcer.models.ApplicationMapping; +import org.wso2.apk.enforcer.models.Subscription; import org.wso2.apk.enforcer.security.KeyValidator; +import org.wso2.apk.enforcer.subscription.SubscriptionDataHolder; +import org.wso2.apk.enforcer.subscription.SubscriptionDataStore; import org.wso2.apk.enforcer.util.BackendJwtUtils; import org.wso2.apk.enforcer.util.FilterUtils; import org.wso2.apk.enforcer.util.JWTUtils; @@ -59,6 +65,9 @@ import java.security.cert.Certificate; import java.text.ParseException; import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Extends the APIKeyHandler to authenticate request using API Key. @@ -213,8 +222,8 @@ private AuthenticationContext processAPIKey(RequestContext requestContext, Strin .getMandateSubscriptionValidation(); if (!requestContext.getMatchedAPI().isSystemAPI() && (isGatewayLevelSubscriptionValidationEnabled || requestContext.getMatchedAPI() .isSubscriptionValidation())) { -// validationInfoDto = KeyValidator.validateSubscription(apiUuid, apiContext, -// requestContext.getMatchedAPI(), payload); + validationInfoDto = KeyValidator.validateSubscription(apiUuid, apiContext, + requestContext.getMatchedAPI(), payload); log.debug("Validating subscription for API Key using JWT claims against invoked API info." + " context: {} version: {}", apiContext, apiVersion); validationInfoDto = getAPIKeyValidationDTO(requestContext, payload); @@ -284,6 +293,32 @@ private AuthenticationContext processAPIKey(RequestContext requestContext, Strin // Set generated jwt token as a response header requestContext.addOrModifyHeaders(jwtConfigurationDto.getJwtHeader(), endUserToken); } + SubscriptionDataStore datastore = SubscriptionDataHolder.getInstance(). + getSubscriptionDataStore(organization); + Application app = getApplication(requestContext.getMatchedAPI(), payload); + Set appMappings = datastore.getMatchingApplicationMappings(app.getUUID()); + for (ApplicationMapping appMapping : appMappings) { + String subscriptionUUID = appMapping.getSubscriptionUUID(); + Subscription subscription = datastore.getMatchingSubscription(subscriptionUUID); + if (requestContext.getMatchedAPI().getName().equals(subscription.getSubscribedApi().getName())) { + // Validate API version + Pattern pattern = subscription.getSubscribedApi().getVersionRegexPattern(); + String versionToMatch = requestContext.getMatchedAPI().getVersion(); + Matcher matcher = pattern.matcher(versionToMatch); + if (matcher.matches()) { + if (!"Unlimited".equals(subscription.getRatelimitTier())) { + String subscriptionId = subscription.getSubscribedApi().getName() + ":" + + app.getUUID(); + requestContext.addMetadataToMap("ratelimit:subscription", subscriptionId); + requestContext.addMetadataToMap("ratelimit:usage-policy", subscription.getRatelimitTier()); + requestContext.addMetadataToMap("ratelimit:organization", subscription.getOrganization()); + System.out.println("Value: "+ String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + requestContext.addMetadataToMap("ratelimit:organization-and-rlpolicy", String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + } + break; + } + } + } // Create authentication context AuthenticationContext authenticationContext = FilterUtils @@ -501,4 +536,23 @@ public int getPriority() { return 30; } + + public static Application getApplication(APIConfig api, JWTClaimsSet payload) { + Application app = null; + + SubscriptionDataStore datastore = + SubscriptionDataHolder.getInstance().getSubscriptionDataStore(api.getOrganizationId()); + if (datastore != null) { + JSONObject appObject = (JSONObject) payload.getClaim(APIConstants.JwtTokenConstants.APPLICATION); + String appUuid = appObject.getAsString("uuid"); + if (!appObject.isEmpty() && !appUuid.isEmpty()) { + app = datastore.getApplicationById(appUuid); + } else { + log.info("Application claim not found in jwt for uuid"); + } + } else { + log.error("Subscription data store is null"); + } + return app; + } } diff --git a/libs.versions.toml b/libs.versions.toml index 64f34aa1d2..09438644cc 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -21,6 +21,7 @@ azure-monitor-opentelemetry = {module = "com.azure:azure-monitor-opentelemetry-e azure-okhttp = {module = "com.azure:azure-core-http-okhttp", version.ref = "azure-okhttp"} commons-codec = {module = "commons-codec:commons-codec", version.ref = "commons-codec"} commons-collection = {module = "org.apache.commons:commons-collections4", version.ref = "commons-collection"} +commons-compress = {module = "org.apache.commons:commons-compress", version.ref = "commons-compress"} commons-io = {module = "commons-io:commons-io", version.ref = "commons-io"} commons-lang = {module = "commons-lang:commons-lang", version.ref = "commons-lang"} commons-lang3 = {module = "org.apache.commons:commons-lang3", version.ref = "commons-lang3"} @@ -124,6 +125,7 @@ azure-monitor-opentelemetry = "1.0.0-beta.9" azure-okhttp = "1.11.6" commons-codec = "1.14" commons-collection = "4.1" +commons-compress = "1.27.1" commons-io = "2.11.0" commons-lang = "2.4" commons-lang3 = "3.10" From ac43de6c8c723066ccd841867b2214e4d746bbcb Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 13 Sep 2024 09:01:56 +0530 Subject: [PATCH 12/24] Improve subscription matching --- .../security/jwt/APIKeyAuthenticator.java | 2 +- .../security/jwt/Oauth2Authenticator.java | 27 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java index 24dcd43518..0eb2a90ef1 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java @@ -308,7 +308,7 @@ private AuthenticationContext processAPIKey(RequestContext requestContext, Strin if (matcher.matches()) { if (!"Unlimited".equals(subscription.getRatelimitTier())) { String subscriptionId = subscription.getSubscribedApi().getName() + ":" + - app.getUUID(); + app.getUUID() + subscription.getSubscriptionId(); requestContext.addMetadataToMap("ratelimit:subscription", subscriptionId); requestContext.addMetadataToMap("ratelimit:usage-policy", subscription.getRatelimitTier()); requestContext.addMetadataToMap("ratelimit:organization", subscription.getOrganization()); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java index b12dda1c0a..a653bc3054 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/Oauth2Authenticator.java @@ -68,6 +68,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Implements the authenticator interface to authenticate request using a JWT token. @@ -262,14 +264,23 @@ public AuthenticationContext authenticate(RequestContext requestContext) throws for (ApplicationMapping appMapping : appMappings) { String subscriptionUUID = appMapping.getSubscriptionUUID(); Subscription subscription = datastore.getMatchingSubscription(subscriptionUUID); - if (!"Unlimited".equals(subscription.getRatelimitTier())) { - String subscriptionId = subscription.getSubscribedApi().getName() + ":" + - applicationId; - requestContext.addMetadataToMap("ratelimit:subscription", subscriptionId); - requestContext.addMetadataToMap("ratelimit:usage-policy", subscription.getRatelimitTier()); - requestContext.addMetadataToMap("ratelimit:organization", subscription.getOrganization()); - System.out.println("Value: "+ String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); - requestContext.addMetadataToMap("ratelimit:organization-and-rlpolicy", String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + if (requestContext.getMatchedAPI().getName().equals(subscription.getSubscribedApi().getName())) { + // Validate API version + Pattern pattern = subscription.getSubscribedApi().getVersionRegexPattern(); + String versionToMatch = requestContext.getMatchedAPI().getVersion(); + Matcher matcher = pattern.matcher(versionToMatch); + if (matcher.matches()) { + if (!"Unlimited".equals(subscription.getRatelimitTier())) { + String subscriptionId = subscription.getSubscribedApi().getName() + ":" + + applicationId + subscription.getSubscriptionId(); + requestContext.addMetadataToMap("ratelimit:subscription", subscriptionId); + requestContext.addMetadataToMap("ratelimit:usage-policy", subscription.getRatelimitTier()); + requestContext.addMetadataToMap("ratelimit:organization", subscription.getOrganization()); + System.out.println("Value: " + String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + requestContext.addMetadataToMap("ratelimit:organization-and-rlpolicy", String.format("%s-%s", subscription.getOrganization(), subscription.getRatelimitTier())); + } + break; + } } } } From 65b5dd109cef7de54a6fcd0deb875ac4f6def117 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 13 Sep 2024 10:22:08 +0530 Subject: [PATCH 13/24] Reduce response path token count by one --- .../apk/enforcer/grpc/ExternalProcessorService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index 7524c30f35..4d776ad9b3 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -114,9 +114,9 @@ public void onNext(ProcessingRequest request) { } List configs = new ArrayList<>(); if (filterMetadata.enableBackendBasedAIRatelimit) { - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens() - 1)); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens() - 1)); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens() - 1)); } if (filterMetadata.enableSubscriptionBasedAIRatelimit) { if (request.hasMetadataContext()) { @@ -124,9 +124,9 @@ public void onNext(ProcessingRequest request) { if (filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY) != null && filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION) != null) { String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens()))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens()))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens() - 1))); } } } From 10e8e076a4e28718b450045558fabbb55c071ab9 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Fri, 13 Sep 2024 16:36:55 +0530 Subject: [PATCH 14/24] Move rl to vhost --- adapter/internal/discovery/xds/server.go | 25 +- .../internal/oasparser/config_generator.go | 10 +- .../internal/oasparser/envoyconf/constants.go | 1 + .../oasparser/envoyconf/http_filters.go | 2 + .../internal/oasparser/envoyconf/listener.go | 314 +++++++++++++++++- .../oasparser/envoyconf/listener_test.go | 4 +- .../oasparser/envoyconf/routes_configs.go | 312 +---------------- .../envoyconf/routes_with_clusters.go | 48 ++- .../envoyconf/routes_with_clusters_test.go | 18 + .../oasparser/model/adapter_internal_api.go | 8 +- adapter/internal/oasparser/model/resource.go | 14 +- .../operator/controllers/dp/api_controller.go | 100 +----- .../operator/synchronizer/api_state.go | 1 - .../operator/synchronizer/data_store.go | 11 - .../operator/synchronizer/rest_api.go | 2 +- .../dp/airatelimitpolicy_controller.go | 39 --- .../internal/xds/ratelimiter_cache.go | 5 - .../org.wso2.apk.enforcer/build.gradle | 1 + .../apk/enforcer/grpc/ExtAuthService.java | 3 - .../grpc/ExternalProcessorService.java | 97 ++++-- .../enforcer/grpc/client/RatelimitClient.java | 10 +- .../apk/enforcer/security/KeyValidator.java | 2 - .../security/jwt/APIKeyAuthenticator.java | 4 +- .../wso2/apk/enforcer/server/AuthServer.java | 1 - .../subscription/SubscriptionDto.java | 2 +- .../gateway-runtime-deployment.yaml | 4 +- 26 files changed, 498 insertions(+), 540 deletions(-) diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index 1f1cd89d26..2c45800c78 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -97,9 +97,10 @@ var ( // todo(amali) there can be multiple vhosts for one EnvoyInternalAPI so handle this apiuuid+sand/prod should be the key - orgAPIMap map[string]map[string]*EnvoyInternalAPI // organizationID -> Vhost:API_UUID -> EnvoyInternalAPI struct map - orgIDLatestAPIVersionMap map[string]map[string]map[string]semantic_version.SemVersion // organizationID -> Vhost:APIName -> VersionRange(vx/vx.x; x is int) -> Latest API Version - + orgAPIMap map[string]map[string]*EnvoyInternalAPI // organizationID -> Vhost:API_UUID -> EnvoyInternalAPI struct map + orgIDLatestAPIVersionMap map[string]map[string]map[string]semantic_version.SemVersion // organizationID -> Vhost:APIName -> VersionRange(vx/vx.x; x is int) -> Latest API Version + vHostToSubscriptionBasedAIRLMap map[string]bool + vHostToSubscriptionBasedRLMap map[string]bool // Envoy Label as map key // TODO(amali) use this without generating all again. gatewayLabelConfigMap map[string]*EnvoyGatewayConfig // GW-Label -> EnvoyGatewayConfig struct map @@ -152,6 +153,8 @@ func init() { gatewayLabelConfigMap = make(map[string]*EnvoyGatewayConfig) orgAPIMap = make(map[string]map[string]*EnvoyInternalAPI) orgIDLatestAPIVersionMap = make(map[string]map[string]map[string]semantic_version.SemVersion) + vHostToSubscriptionBasedAIRLMap = make(map[string]bool) + vHostToSubscriptionBasedRLMap = make(map[string]bool) enforcerLabelMap = make(map[string]*EnforcerInternalAPI) // currently subscriptions, configs, applications, applicationPolicies, subscriptionPolicies, @@ -330,7 +333,7 @@ func GenerateEnvoyResoucesForGateway(gatewayName string) ([]types.Resource, if found { // Prepare the route config name based on the gateway listener section name. routeConfigName := common.GetEnvoyRouteConfigName(listener.Name, string(listenerSection.Name)) - routesConfig := oasParser.GetRouteConfigs(map[string][]*routev3.Route{vhost: routes}, routeConfigName, envoyGatewayConfig.customRateLimitPolicies) + routesConfig := oasParser.GetRouteConfigs(map[string][]*routev3.Route{vhost: routes}, routeConfigName, envoyGatewayConfig.customRateLimitPolicies, vHostToSubscriptionBasedAIRLMap, vHostToSubscriptionBasedRLMap) routeConfigMatched, alreadyExistsInRouteConfigList := routeConfigs[routeConfigName] if alreadyExistsInRouteConfigList { @@ -353,7 +356,7 @@ func GenerateEnvoyResoucesForGateway(gatewayName string) ([]types.Resource, var vhostToRouteArrayFilteredMapForSystemEndpoints = make(map[string][]*routev3.Route) vhostToRouteArrayFilteredMapForSystemEndpoints[systemHost] = vhostToRouteArrayMap[systemHost] routeConfigName := common.GetEnvoyRouteConfigName(common.GetEnvoyListenerName(string(listener.Protocol), uint32(listener.Port)), string(listener.Name)) - systemRoutesConfig := oasParser.GetRouteConfigs(vhostToRouteArrayFilteredMapForSystemEndpoints, routeConfigName, envoyGatewayConfig.customRateLimitPolicies) + systemRoutesConfig := oasParser.GetRouteConfigs(vhostToRouteArrayFilteredMapForSystemEndpoints, routeConfigName, envoyGatewayConfig.customRateLimitPolicies, vHostToSubscriptionBasedAIRLMap, vHostToSubscriptionBasedRLMap) routeConfigs[routeConfigName] = systemRoutesConfig } } @@ -560,6 +563,18 @@ func PopulateInternalMaps(adapterInternalAPI *model.AdapterInternalAPI, labels, } err := UpdateOrgAPIMap(vHosts, labels, listenerName, sectionName, adapterInternalAPI) + for vhost := range vHosts { + if adapterInternalAPI.AIProvider.Enabled && adapterInternalAPI.GetSubscriptionValidation() { + vHostToSubscriptionBasedAIRLMap[vhost] = true + } else { + vHostToSubscriptionBasedAIRLMap[vhost] = false + } + if adapterInternalAPI.GetSubscriptionValidation() { + vHostToSubscriptionBasedRLMap[vhost] = true + } else { + vHostToSubscriptionBasedRLMap[vhost] = false + } + } if err != nil { logger.LoggerXds.ErrorC(logging.PrintError(logging.Error1415, logging.MAJOR, "Error updating the API : %s:%s in vhosts: %s, API_UUID: %v. %v", diff --git a/adapter/internal/oasparser/config_generator.go b/adapter/internal/oasparser/config_generator.go index eb4721e7cc..807fdd9cbc 100644 --- a/adapter/internal/oasparser/config_generator.go +++ b/adapter/internal/oasparser/config_generator.go @@ -85,8 +85,8 @@ func GetProductionListener(gateway *gwapiv1.Gateway, resolvedListenerCerts map[s // // The RouteConfiguration is named as "default" func GetRouteConfigs(vhostToRouteArrayMap map[string][]*routev3.Route, routeConfigName string, - customRateLimitPolicies []*model.CustomRateLimitPolicy) *routev3.RouteConfiguration { - vHosts := envoy.CreateVirtualHosts(vhostToRouteArrayMap, customRateLimitPolicies) + customRateLimitPolicies []*model.CustomRateLimitPolicy, vhostToSubscriptionAIRL map[string]bool, vhostToSubscriptionRL map[string]bool) *routev3.RouteConfiguration { + vHosts := envoy.CreateVirtualHosts(vhostToRouteArrayMap, customRateLimitPolicies, vhostToSubscriptionAIRL, vhostToSubscriptionRL) routeConfig := envoy.CreateRoutesConfigForRds(vHosts, routeConfigName) return routeConfig } @@ -119,12 +119,6 @@ func GetCacheResources(endpoints []*corev3.Address, clusters []*clusterv3.Cluste return listenerRes, clusterRes, routeConfigRes, endpointRes } -// UpdateRoutesConfig updates the existing routes configuration with the provided map of vhost to array of routes. -// All the already existing routes (within the routeConfiguration) will be removed. -func UpdateRoutesConfig(routeConfig *routev3.RouteConfiguration, vhostToRouteArrayMap map[string][]*routev3.Route) { - routeConfig.VirtualHosts = envoy.CreateVirtualHosts(vhostToRouteArrayMap, nil) -} - // GetEnforcerAPI retrieves the ApiDS object model for a given swagger definition // along with the vhost to deploy the API. func GetEnforcerAPI(adapterInternalAPI *model.AdapterInternalAPI, vhost string) *api.Api { diff --git a/adapter/internal/oasparser/envoyconf/constants.go b/adapter/internal/oasparser/envoyconf/constants.go index c92501c054..04a92f34ab 100644 --- a/adapter/internal/oasparser/envoyconf/constants.go +++ b/adapter/internal/oasparser/envoyconf/constants.go @@ -30,6 +30,7 @@ const ( httpConManagerStartPrefix string = "ingress_http" extAuthzPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute" extProcPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute" + ratelimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimitPerRoute" luaPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute" corsFilterName string = "type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors" localRateLimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit" diff --git a/adapter/internal/oasparser/envoyconf/http_filters.go b/adapter/internal/oasparser/envoyconf/http_filters.go index 4017426f87..cedd7784bd 100644 --- a/adapter/internal/oasparser/envoyconf/http_filters.go +++ b/adapter/internal/oasparser/envoyconf/http_filters.go @@ -51,6 +51,8 @@ import ( // HTTPExternalProcessor HTTP filter const HTTPExternalProcessor = "envoy.filters.http.ext_proc" +// RatelimitFilterName Ratelimit filter name +const RatelimitFilterName = "envoy.filters.http.ratelimit" // getHTTPFilters generates httpFilter configuration func getHTTPFilters(globalLuaScript string) []*hcmv3.HttpFilter { diff --git a/adapter/internal/oasparser/envoyconf/listener.go b/adapter/internal/oasparser/envoyconf/listener.go index 7237a13d24..493550539f 100644 --- a/adapter/internal/oasparser/envoyconf/listener.go +++ b/adapter/internal/oasparser/envoyconf/listener.go @@ -334,7 +334,7 @@ func CreateListenerByGateway(gateway *gwapiv1.Gateway, resolvedListenerCerts map // CreateVirtualHosts creates VirtualHost configurations for envoy which serves // request from the vHost domain. The routes array will be included as the routes // for the created virtual host. -func CreateVirtualHosts(vhostToRouteArrayMap map[string][]*routev3.Route, customRateLimitPolicies []*model.CustomRateLimitPolicy) []*routev3.VirtualHost { +func CreateVirtualHosts(vhostToRouteArrayMap map[string][]*routev3.Route, customRateLimitPolicies []*model.CustomRateLimitPolicy, vhostToSubscriptionAIRL map[string]bool, vhostToSubscriptionRL map[string]bool) []*routev3.VirtualHost { virtualHosts := make([]*routev3.VirtualHost, 0, len(vhostToRouteArrayMap)) var rateLimits []*routev3.RateLimit for _, customRateLimitPolicy := range customRateLimitPolicies { @@ -381,6 +381,12 @@ func CreateVirtualHosts(vhostToRouteArrayMap map[string][]*routev3.Route, custom } for vhost, routes := range vhostToRouteArrayMap { + if flag, exists := vhostToSubscriptionAIRL[vhost]; exists && flag { + rateLimits = append(rateLimits, generateSubscriptionBasedAIRatelimits()...) + } + if flag, exists := vhostToSubscriptionRL[vhost]; exists && flag { + rateLimits = append(rateLimits, generateSubscriptionBasedRatelimits()...) + } virtualHost := &routev3.VirtualHost{ Name: vhost, Domains: []string{vhost, fmt.Sprint(vhost, ":*")}, @@ -392,6 +398,312 @@ func CreateVirtualHosts(vhostToRouteArrayMap map[string][]*routev3.Route, custom return virtualHosts } + +func generateSubscriptionBasedRatelimits() []*routev3.RateLimit { + return []*routev3.RateLimit{&routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForOrganization, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForOrganization, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForSubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForPolicy, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForUsagePolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + }, &routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForOrganization, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForOrganization, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForSubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForPolicy, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + { + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: descriptorMetadataKeyForUsagePolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: "burst", + DescriptorValue: "enabled", + }, + }, + }, + }, + }, + } +} + +func generateSubscriptionBasedAIRatelimits() []*routev3.RateLimit { + rateLimitForRequestTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestTokenCountForSubscriptionBasedAIRL, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + } + rateLimitForResponseTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIResponseTokenCountForSubscriptionBasedAIRL, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + } + rateLimitForRequestCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestCountForSubscriptionBasedAIRL, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + } + rateLimitForTotalTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAITotalTokenCountForSubscriptionBasedAIRL, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAISubscription, + MetadataKey: &metadatav3.MetadataKey{ + Key: extAuthzFilterName, + Path: []*metadatav3.MetadataKey_PathSegment{ + &metadatav3.MetadataKey_PathSegment{ + Segment: &metadatav3.MetadataKey_PathSegment_Key{ + Key: DynamicMetadataKeyForSubscription, + }, + }, + }, + }, + Source: routev3.RateLimit_Action_MetaData_DYNAMIC, + SkipIfAbsent: true, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount, &rateLimitForTotalTokenCount} +} + // TODO: (VirajSalaka) Still the following method is not utilized as Sds is not implement. Keeping the Implementation for future reference // func generateDefaultSdsSecretFromConfigfile(privateKeyPath string, pulicKeyPath string) (*tlsv3.Secret, error) { // var secret tlsv3.Secret diff --git a/adapter/internal/oasparser/envoyconf/listener_test.go b/adapter/internal/oasparser/envoyconf/listener_test.go index 38308b8549..cf17e636e7 100644 --- a/adapter/internal/oasparser/envoyconf/listener_test.go +++ b/adapter/internal/oasparser/envoyconf/listener_test.go @@ -66,7 +66,7 @@ func TestCreateVirtualHost(t *testing.T) { "*": testCreateRoutesForUnitTests(t), "mg.wso2.com": testCreateRoutesForUnitTests(t), } - vHosts := CreateVirtualHosts(vhostToRouteArrayMap, nil) + vHosts := CreateVirtualHosts(vhostToRouteArrayMap, nil, make(map[string]bool), make(map[string]bool)) if len(vHosts) != 2 { t.Error("Virtual Host creation failed") @@ -92,7 +92,7 @@ func TestCreateRoutesConfigForRds(t *testing.T) { "mg.wso2.com": testCreateRoutesForUnitTests(t), } httpListeners := "httpslistener" - vHosts := CreateVirtualHosts(vhostToRouteArrayMap, nil) + vHosts := CreateVirtualHosts(vhostToRouteArrayMap, nil, make(map[string]bool), make(map[string]bool)) rConfig := CreateRoutesConfigForRds(vHosts, httpListeners) assert.NotNil(t, rConfig, "CreateRoutesConfigForRds is failed") diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index 4798a48cd5..7b005904bc 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -29,7 +29,6 @@ import ( extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" envoy_type_matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" - metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/golang/protobuf/ptypes/any" logger "github.com/wso2/apk/adapter/internal/loggers" @@ -41,7 +40,6 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - v35 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" ) // Constants for Rate Limiting @@ -118,7 +116,7 @@ func generateRouteMatch(routeRegex string) *routev3.RouteMatch { return match } -func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string, isSubscriptionBasedAIRatelimitEnabled bool, isBackendBasedAIRatelimitEnabled bool, descriptorValueForBackendBasedAIRatelimit string) (action *routev3.Route_Route) { +func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string, isBackendBasedAIRatelimitEnabled bool, descriptorValueForBackendBasedAIRatelimit string) (action *routev3.Route_Route) { action = &routev3.Route_Route{ Route: &routev3.RouteAction{ HostRewriteSpecifier: &routev3.RouteAction_AutoHostRewrite{ @@ -152,9 +150,6 @@ func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, rate if isBackendBasedAIRatelimitEnabled { action.Route.RateLimits = append(action.Route.RateLimits, generateBackendBasedAIRatelimit(descriptorValueForBackendBasedAIRatelimit)...) } - if isSubscriptionBasedAIRatelimitEnabled { - action.Route.RateLimits = append(action.Route.RateLimits, generateSubscriptionBasedAIRatelimit()...) - } // Add request mirroring configurations if mirrorClusterNames != nil && len(mirrorClusterNames) > 0 { @@ -263,177 +258,7 @@ func generateBackendBasedAIRatelimit(descValue string) []*routev3.RateLimit { } -func generateSubscriptionBasedAIRatelimit() []*routev3.RateLimit { - rateLimitForRequestTokenCount := routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIRequestTokenCountForSubscriptionBasedAIRL, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAISubscription, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - }, - } - rateLimitForResponseTokenCount := routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIResponseTokenCountForSubscriptionBasedAIRL, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAISubscription, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - }, - } - rateLimitForRequestCount := routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAIRequestCountForSubscriptionBasedAIRL, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAISubscription, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - }, - } - rateLimitForTotalTokenCount := routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAITotalTokenCountForSubscriptionBasedAIRL, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForOrganizationAndAIRLPolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForAISubscription, - MetadataKey: &v35.MetadataKey{ - Key: extAuthzFilterName, - Path: []*v35.MetadataKey_PathSegment{ - &v35.MetadataKey_PathSegment{ - Segment: &v35.MetadataKey_PathSegment_Key{ - Key: DynamicMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - }, - } - return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount, &rateLimitForTotalTokenCount} -} + func generateRateLimitPolicy(ratelimitCriteria *ratelimitCriteria) []*routev3.RateLimit { environmentValue := ratelimitCriteria.environment @@ -492,7 +317,6 @@ func generateRateLimitPolicy(ratelimitCriteria *ratelimitCriteria) []*routev3.Ra } ratelimits := []*routev3.RateLimit{&rateLimit} - ratelimits = addSubscriptionRatelimitActions(ratelimits) return ratelimits } @@ -533,138 +357,6 @@ func generateHeaderMatcher(headerName, valueRegex string) *routev3.HeaderMatcher return headerMatcherArray } -func addSubscriptionRatelimitActions(actions []*routev3.RateLimit) []*routev3.RateLimit { - return append(actions, - &routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForOrganization, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForOrganization, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForSubscription, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForPolicy, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForUsagePolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - }, - }, &routev3.RateLimit{ - Actions: []*routev3.RateLimit_Action{ - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForOrganization, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForOrganization, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForSubscription, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForSubscription, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_Metadata{ - Metadata: &routev3.RateLimit_Action_MetaData{ - DescriptorKey: DescriptorKeyForPolicy, - MetadataKey: &metadatav3.MetadataKey{ - Key: extAuthzFilterName, - Path: []*metadatav3.MetadataKey_PathSegment{ - { - Segment: &metadatav3.MetadataKey_PathSegment_Key{ - Key: descriptorMetadataKeyForUsagePolicy, - }, - }, - }, - }, - Source: routev3.RateLimit_Action_MetaData_DYNAMIC, - SkipIfAbsent: true, - }, - }, - }, - { - ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ - GenericKey: &routev3.RateLimit_Action_GenericKey{ - DescriptorKey: "burst", - DescriptorValue: "enabled", - }, - }, - }, - }, - }) -} func generateRegexMatchAndSubstitute(routePath, endpointResourcePath string, pathMatchType gwapiv1.PathMatchType) *envoy_type_matcherv3.RegexMatchAndSubstitute { diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index a4fb91b1c8..772031f383 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -40,6 +40,7 @@ import ( cors_filter_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" + ratelimitv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ratelimit/v3" lua "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" upstreams "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" @@ -865,7 +866,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error LuaLocal: luaFilter, wellknown.CORS: corsFilter, } - if !params.isAiAPI || (!resource.GetEnableBackendBasedAIRatelimit() && !resource.GetEnableSubscriptionBasedAIRatelimit()) { + if !params.isAiAPI { perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ Override: &extProcessorv3.ExtProcPerRoute_Disabled{ Disabled: true, @@ -877,7 +878,36 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error Value: dataExtProc, } perRouteFilterConfigs[HTTPExternalProcessor] = filterExtProc + } else { + if strings.ToUpper(resource.GetExtractTokenFromValue()) == "HEADER" { + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Overrides{ + Overrides: &extProcessorv3.ExtProcOverrides{ + ProcessingMode: &extProcessorv3.ProcessingMode{ + RequestHeaderMode: extProcessorv3.ProcessingMode_SKIP, + ResponseHeaderMode: extProcessorv3.ProcessingMode_SKIP, + ResponseBodyMode: extProcessorv3.ProcessingMode_NONE, + }, + }, + }, + } + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + perRouteFilterConfigs[HTTPExternalProcessor] = filterExtProc + } + } + perFilterConfigRL := ratelimitv3.RateLimitPerRoute{ + VhRateLimits: ratelimitv3.RateLimitPerRoute_INCLUDE, + } + ratelimitPerRoute, _ := proto.Marshal(&perFilterConfigRL) + filterrl := &any.Any{ + TypeUrl: ratelimitPerRouteName, + Value: ratelimitPerRoute, } + perRouteFilterConfigs[RatelimitFilterName] = filterrl logger.LoggerOasparser.Debugf("adding route : %s for API : %s", resourcePath, title) @@ -930,8 +960,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } routeConfig := resource.GetEndpoints().Config metaData := &corev3.Metadata{} - logger.LoggerAPI.Infof("Is backend based rl enabled: %+v, Is subs based rl enabled: %+v", resource.GetEnableBackendBasedAIRatelimit(), resource.GetEnableSubscriptionBasedAIRatelimit()) - if params.isAiAPI && (resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit()) { + if params.isAiAPI { metaData = &corev3.Metadata{ FilterMetadata: map[string]*structpb.Struct{ "envoy.filters.http.ext_proc": &structpb.Struct{ @@ -941,11 +970,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error StringValue: fmt.Sprintf("%t", resource.GetEnableBackendBasedAIRatelimit()), }, }, - "EnableSubscriptionBasedAIRatelimit": &structpb.Value{ - Kind: &structpb.Value_StringValue{ - StringValue: fmt.Sprintf("%t", resource.GetEnableSubscriptionBasedAIRatelimit()), - }, - }, "BackendBasedAIRatelimitDescriptorValue": &structpb.Value{ Kind: &structpb.Value_StringValue{ StringValue: resource.GetBackendBasedAIRatelimitDescriptorValue(), @@ -1092,8 +1116,8 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error metadataValue := operation.GetMethod() + "_to_" + newMethod match2.DynamicMetadata = generateMetadataMatcherForInternalRoutes(metadataValue) - action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) - action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) + action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) + action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) requestHeadersToRemove := make([]string,0) if params.isAiAPI { @@ -1125,7 +1149,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else { var action *routev3.Route_Route if requestRedirectAction == nil { - action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit() && params.isAiAPI, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) + action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) } logger.LoggerOasparser.Debug("Creating routes for resource with policies", resourcePath, operation.GetMethod()) // create route for current method. Add policies to route config. Send via enforcer @@ -1154,7 +1178,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } match := generateRouteMatch(routePath) match.Headers = generateHTTPMethodMatcher(methodRegex, clusterName) - action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) + action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) requestHeadersToRemove := make([]string,0) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go index f914c68b81..ab26811a40 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters_test.go @@ -31,6 +31,7 @@ import ( operatorutils "github.com/wso2/apk/adapter/internal/operator/utils" "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8types "k8s.io/apimachinery/pkg/types" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -131,6 +132,8 @@ func TestCreateRoutesWithClustersWithExactAndRegularExpressionRules(t *testing.T httpRouteState.BackendMapping = backendMapping apiState.ProdHTTPRoute = &httpRouteState + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) adapterInternalAPI, labels, err := synchronizer.UpdateInternalMapsFromHTTPRoute(apiState, &httpRouteState, constants.Production) assert.Equal(t, map[string]struct{}{"default-gateway": {}}, labels, "Labels are incorrect.") assert.Nil(t, err, "Error should not be present when apiState is converted to a AdapterInternalAPI object") @@ -186,6 +189,8 @@ func TestExtractAPIDetailsFromHTTPRouteForDefaultCase(t *testing.T) { apiState := generateSampleAPI("test-api-1", "1.0.0", "/test-api/1.0.0") httpRouteState := synchronizer.HTTPRouteState{} httpRouteState = *apiState.ProdHTTPRoute + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) xds.SanitizeGateway("default-gateway", true) adapterInternalAPI, labels, err := synchronizer.UpdateInternalMapsFromHTTPRoute(apiState, &httpRouteState, constants.Production) assert.Equal(t, map[string]struct{}{"default-gateway": {}}, labels, "Labels are incorrect.") @@ -201,6 +206,8 @@ func TestExtractAPIDetailsFromHTTPRouteForSpecificEnvironment(t *testing.T) { apiState.APIDefinition.Spec.Environment = "dev" xds.SanitizeGateway("default-gateway", true) + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) adapterInternalAPI, labels, err := synchronizer.UpdateInternalMapsFromHTTPRoute(apiState, &httpRouteState, constants.Production) assert.Equal(t, map[string]struct{}{"default-gateway": {}}, labels, "Labels are incorrect.") assert.Nil(t, err, "Error should not be present when apiState is converted to a AdapterInternalAPI object") @@ -267,6 +274,9 @@ func generateSampleAPI(apiName string, apiVersion string, basePath string) synch httpRouteState.BackendMapping = backendMapping apiState.ProdHTTPRoute = &httpRouteState + + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) return apiState } @@ -294,6 +304,8 @@ func TestCreateRoutesWithClustersWithMultiplePathPrefixRules(t *testing.T) { apiState.APIDefinition = &apiDefinition httpRouteState := synchronizer.HTTPRouteState{} + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) httpRoute := gwapiv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -444,6 +456,8 @@ func TestCreateRoutesWithClustersWithBackendTLSConfigs(t *testing.T) { httpRouteState := synchronizer.HTTPRouteState{} methodTypeGet := gwapiv1.HTTPMethodGet + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) httpRoute := gwapiv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -567,6 +581,8 @@ func TestCreateRoutesWithClustersDifferentBackendRefs(t *testing.T) { httpRouteState := synchronizer.HTTPRouteState{} methodTypeGet := gwapiv1.HTTPMethodGet + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) httpRoute := gwapiv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -659,6 +675,8 @@ func TestCreateRoutesWithClustersSameBackendRefs(t *testing.T) { httpRouteState := synchronizer.HTTPRouteState{} methodTypeGet := gwapiv1.HTTPMethodGet + apiState.AIProvider = new(v1alpha3.AIProvider) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = make(map[int]*v1alpha3.AIRateLimitPolicy) httpRoute := gwapiv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 9dabb2f622..0a50bbff3f 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -501,7 +501,7 @@ func (adapterInternalAPI *AdapterInternalAPI) Validate() error { // SetInfoHTTPRouteCR populates resources and endpoints of adapterInternalAPI. httpRoute.Spec.Rules.Matches // are used to create resources and httpRoute.Spec.Rules.BackendRefs are used to create EndpointClusters. -func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams, isAiSubscriptionRatelimitEnabled bool, ruleIdxToAiRatelimitPolicyMapping map[int]*dpv1alpha3.AIRateLimitPolicy) error { +func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams, ruleIdxToAiRatelimitPolicyMapping map[int]*dpv1alpha3.AIRateLimitPolicy, extractTokenFrom string) error { var resources []*Resource outputAuthScheme := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.AuthSchemes))) outputAPIPolicy := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.APIPolicies))) @@ -548,11 +548,11 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap enableBackendBasedAIRatelimit := false descriptorValue := "" if aiRatelimitPolicy, exists := ruleIdxToAiRatelimitPolicyMapping[ruleID]; exists { - loggers.LoggerAPI.Infof("Found AI ratelimit mapping for ruleId: %d, related api: %s", ruleID, adapterInternalAPI.UUID) + loggers.LoggerAPI.Debugf("Found AI ratelimit mapping for ruleId: %d, related api: %s", ruleID, adapterInternalAPI.UUID) enableBackendBasedAIRatelimit = true descriptorValue = prepareAIRatelimitIdentifier(adapterInternalAPI.OrganizationID, utils.NamespacedName(aiRatelimitPolicy), &aiRatelimitPolicy.Spec) } else { - loggers.LoggerAPI.Infof("Could not find AIratelimit for ruleId: %d, len of map: %d, related api: %s", ruleID, len(ruleIdxToAiRatelimitPolicyMapping), adapterInternalAPI.UUID) + loggers.LoggerAPI.Debugf("Could not find AIratelimit for ruleId: %d, len of map: %d, related api: %s", ruleID, len(ruleIdxToAiRatelimitPolicyMapping), adapterInternalAPI.UUID) } backendBasePath := "" @@ -919,9 +919,9 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap hasPolicies: true, iD: uuid.New().String(), hasRequestRedirectFilter: hasRequestRedirectPolicy, - enableSubscriptionBasedAIRatelimit: isAiSubscriptionRatelimitEnabled, enableBackendBasedAIRatelimit: enableBackendBasedAIRatelimit, backendBasedAIRatelimitDescriptorValue: descriptorValue, + extractTokenFrom: extractTokenFrom, } resource.endpoints = &EndpointCluster{ diff --git a/adapter/internal/oasparser/model/resource.go b/adapter/internal/oasparser/model/resource.go index a953e3a850..f4137b850f 100644 --- a/adapter/internal/oasparser/model/resource.go +++ b/adapter/internal/oasparser/model/resource.go @@ -45,9 +45,9 @@ type Resource struct { vendorExtensions map[string]interface{} hasPolicies bool hasRequestRedirectFilter bool - enableSubscriptionBasedAIRatelimit bool enableBackendBasedAIRatelimit bool backendBasedAIRatelimitDescriptorValue string + extractTokenFrom string } // GetEndpointSecurity returns the endpoint security object of a given resource. @@ -193,11 +193,6 @@ func SortResources(resources []*Resource) []*Resource { return resources } -// GetEnableSubscriptionBasedAIRatelimit returns the value of enableSubscriptionBasedAIRatelimit. -func (resource *Resource) GetEnableSubscriptionBasedAIRatelimit() bool { - return resource.enableSubscriptionBasedAIRatelimit -} - // GetEnableBackendBasedAIRatelimit returns the value of enableBackendBasedAIRatelimit. func (resource *Resource) GetEnableBackendBasedAIRatelimit() bool { return resource.enableBackendBasedAIRatelimit @@ -206,4 +201,9 @@ func (resource *Resource) GetEnableBackendBasedAIRatelimit() bool { // GetBackendBasedAIRatelimitDescriptorValue returns the value of backendBasedAIRatelimitDescriptorValue. func (resource *Resource) GetBackendBasedAIRatelimitDescriptorValue() string { return resource.backendBasedAIRatelimitDescriptorValue -} \ No newline at end of file +} + +// GetExtractTokenFromValue returns the value of extractTokenFrom +func (resource *Resource) GetExtractTokenFromValue() string { + return resource.extractTokenFrom +} diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index 4ccc105df3..11f40d58e2 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -58,7 +58,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" k8client "sigs.k8s.io/controller-runtime/pkg/client" - cpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" @@ -232,12 +231,6 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera return err } - if err := c.Watch(source.Kind(mgr.GetCache(), &cpv1alpha3.Subscription{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForSubscription), - predicates...); err != nil { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching Subscription resources: %v", err)) - return err - } - loggers.LoggerAPKOperator.Info("API Controller successfully started. Watching API Objects....") go apiReconciler.handleStatus() go apiReconciler.handleLabels(ctx) @@ -399,7 +392,6 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 apiRef.String(), namespace, string(api.ObjectMeta.UID), "api definition file not found") } } - apiReconciler.resolveAiSubscriptionRatelimitPolicies(ctx, apiState) if len(apiState.Authentications) > 0 { if apiState.MutualSSL, err = apiReconciler.resolveAuthentications(ctx, apiState.Authentications); err != nil { return nil, fmt.Errorf("error while resolving authentication %v in namespace: %s was not found. %s", @@ -854,30 +846,6 @@ func (apiReconciler *APIReconciler) getAPIPolicyChildrenRefs(ctx context.Context return interceptorServices, backendJWTs, subscriptionValidation, aiProvider, nil } -func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx context.Context, apiState *synchronizer.APIState) { - apiState.IsAiSubscriptionRatelimitEnabled = false - subscriptionList := &cpv1alpha3.SubscriptionList{} - if err := apiReconciler.client.List(ctx, subscriptionList, &k8client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(subscriptionToAPIIndex, utils.GetSubscriptionToAPIIndexID(apiState.APIDefinition.Spec.APIName, apiState.APIDefinition.Spec.APIVersion)), - }); err != nil { - loggers.LoggerAPKOperator.Infof("No associated subscription found for API: %s", utils.NamespacedName(apiState.APIDefinition)) - return - } - for _, subscription := range subscriptionList.Items { - aiRatelimitPolicy := &dpv1alpha3.AIRateLimitPolicy{} - nn:= types.NamespacedName{ - Name: subscription.Spec.RatelimitRef.Name, - Namespace: subscription.GetNamespace(), - } - if err := apiReconciler.client.Get(ctx, nn, aiRatelimitPolicy, ); err == nil { - loggers.LoggerAPKOperator.Infof("API state set as AI subscription enabled") - apiState.IsAiSubscriptionRatelimitEnabled = true - break - } - loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) - } -} - func (apiReconciler *APIReconciler) resolveAuthentications(ctx context.Context, authentications map[string]dpv1alpha2.Authentication) (*dpv1alpha2.MutualSSL, error) { resolvedMutualSSL := dpv1alpha2.MutualSSL{} @@ -909,10 +877,10 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte if err := apiReconciler.client.List(ctx, aiRLPolicyList, &k8client.ListOptions{ FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToBackendIndex, backendNamespacedName.String()), }); err != nil { - loggers.LoggerAPKOperator.Infof("No associated AI ratelimit policy found for : %s", backendNamespacedName.String()) + loggers.LoggerAPKOperator.Debugf("No associated AI ratelimit policy found for : %s", backendNamespacedName.String()) } else { for _, aiRLPolicy := range aiRLPolicyList.Items { - loggers.LoggerAPKOperator.Infof("Adding mapping for ruleid: %d to aiRLPolicy: %s", id, utils.NamespacedName(&aiRLPolicy)) + loggers.LoggerAPKOperator.Debugf("Adding mapping for ruleid: %d to aiRLPolicy: %s", id, utils.NamespacedName(&aiRLPolicy)) ruleIdxToAiRatelimitPolicyMapping[id] = &aiRLPolicy } } @@ -1012,14 +980,6 @@ func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAIRatelimitPo return requests } -func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { - requests := apiReconciler.getAPIsForSubscription(ctx, obj) - // if len(requests) > 0 { - // apiReconciler.handleOwnerReference(ctx, obj, &requests) - // } - return requests -} - func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAuthentication(ctx context.Context, obj k8client.Object) []reconcile.Request { requests := apiReconciler.getAPIsForAuthentication(ctx, obj) if len(requests) > 0 { @@ -1482,35 +1442,6 @@ func (apiReconciler *APIReconciler) getAPIsForAIRatelimitPolicy(ctx context.Cont return []reconcile.Request{} } -// getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected -// in subscription resources. -func (apiReconciler *APIReconciler) getAPIsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { - subscription, ok := obj.(*cpv1alpha3.Subscription) - if !ok { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) - return []reconcile.Request{} - } - apiList := &dpv1alpha2.APIList{} - if err := apiReconciler.client.List(ctx, apiList, &k8client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(apiToSubscriptionIndex, utils.GetSubscriptionToAPIIndexID(subscription.Spec.API.Name, subscription.Spec.API.Version)), - }); err != nil { - loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2649, logging.CRITICAL, "Unable to find associated APIs for subscription: %s, error: %v", utils.NamespacedName(subscription).String(), err.Error())) - return []reconcile.Request{} - } - requests := []reconcile.Request{} - for _, api := range apiList.Items { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: api.Name, - Namespace: api.Namespace}, - } - requests = append(requests, req) - loggers.LoggerAPKOperator.Infof("Adding reconcile request for API: %s/%s with API UUID: %v due to change in subscription: %v", api.Namespace, api.Name, - string(api.ObjectMeta.UID), utils.NamespacedName(subscription).String()) - } - return requests -} - // getAPIForAuthentication triggers the API controller reconcile method based on the changes detected // from Authentication objects. If the changes are done for an API stored in the Operator Data store, // a new reconcile event will be created and added to the reconcile event queue. @@ -2082,33 +2013,6 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } - // // AIRatelimitPolicy to Subscription indexer - // if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToSubscriptionIndex, - // func(rawObj k8client.Object) []string { - // aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) - // var subscriptions []string - // namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) - // subscriptions = append(subscriptions, types.NamespacedName{ - // Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), - // Namespace: namespace, - // }.String()) - // return subscriptions - // }); err != nil { - // return err - // } - - // Subscription to API indexer - if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha3.Subscription{}, subscriptionToAPIIndex, - func(rawObj k8client.Object) []string { - subscription := rawObj.(*cpv1alpha3.Subscription) - var subscriptions []string - subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", subscription.Spec.API.Name, subscription.Spec.API.Version) - subscriptions = append(subscriptions, subscriptionIdentifierForIndex) - return subscriptions - }); err != nil { - return err - } - // API to Subscription indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.API{}, apiToSubscriptionIndex, func(rawObj k8client.Object) []string { diff --git a/adapter/internal/operator/synchronizer/api_state.go b/adapter/internal/operator/synchronizer/api_state.go index 8536cd62af..cacf066181 100644 --- a/adapter/internal/operator/synchronizer/api_state.go +++ b/adapter/internal/operator/synchronizer/api_state.go @@ -45,7 +45,6 @@ type APIState struct { APIDefinitionFile []byte SubscriptionValidation bool MutualSSL *v1alpha2.MutualSSL - IsAiSubscriptionRatelimitEnabled bool } // HTTPRouteState holds the state of the deployed httpRoutes. This state is compared with diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 8d8f4bb50b..995aff427f 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -75,11 +75,6 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced var updated bool events := []string{} cachedAPI := ods.apiStore[apiNamespacedName] - if cachedAPI.IsAiSubscriptionRatelimitEnabled != apiState.IsAiSubscriptionRatelimitEnabled { - cachedAPI.IsAiSubscriptionRatelimitEnabled = apiState.IsAiSubscriptionRatelimitEnabled - updated = true - events = append(events, "Subscription based AI RatelimitPolicy") - } if cachedAPI.AIProvider == nil && apiState.AIProvider != nil { cachedAPI.AIProvider = apiState.AIProvider @@ -114,18 +109,15 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced for key, aiRl := range apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { if cachedAIRl, exists := cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { - loggers.LoggerAPI.Infof("Returning true * %s %s %d %d", utils.NamespacedName(cachedAIRl).String(), utils.NamespacedName(aiRl).String(), cachedAIRl.Generation, aiRl.Generation) updated = true break } } else { - loggers.LoggerAPI.Info("Returning true&&") updated = true break } } if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { - loggers.LoggerAPI.Info("Returning true ***") updated = true } } @@ -169,18 +161,15 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced for key, aiRl := range apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { if cachedAIRl, exists := cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { - loggers.LoggerAPI.Info("Returning true") updated = true break } } else { - loggers.LoggerAPI.Info("Returning true") updated = true break } } if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { - loggers.LoggerAPI.Info("Returning true") updated = true } } diff --git a/adapter/internal/operator/synchronizer/rest_api.go b/adapter/internal/operator/synchronizer/rest_api.go index 8c82aaa486..cbd8c84e7a 100644 --- a/adapter/internal/operator/synchronizer/rest_api.go +++ b/adapter/internal/operator/synchronizer/rest_api.go @@ -107,7 +107,7 @@ func generateAdapterInternalAPI(apiState APIState, httpRouteState *HTTPRouteStat RateLimitPolicies: apiState.RateLimitPolicies, ResourceRateLimitPolicies: apiState.ResourceRateLimitPolicies, } - if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRouteState.HTTPRouteCombined, resourceParams, apiState.IsAiSubscriptionRatelimitEnabled, httpRouteState.RuleIdxToAiRatelimitPolicyMapping); err != nil { + if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRouteState.HTTPRouteCombined, resourceParams, httpRouteState.RuleIdxToAiRatelimitPolicyMapping, apiState.AIProvider.Spec.RateLimitFields.PromptTokens.In); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2631, logging.MAJOR, "Error setting HttpRoute CR info to adapterInternalAPI. %v", err)) return nil, err } diff --git a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go index 0d618a2277..4791376a32 100644 --- a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go +++ b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go @@ -26,27 +26,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" - // "context" - // "fmt" - // "time" - - // logger "github.com/sirupsen/logrus" - // k8error "k8s.io/apimachinery/pkg/api/errors" - // "k8s.io/apimachinery/pkg/fields" - // "k8s.io/apimachinery/pkg/runtime" - // "k8s.io/apimachinery/pkg/types" - // ctrl "sigs.k8s.io/controller-runtime" - // "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - // "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" - // "sigs.k8s.io/controller-runtime/pkg/reconcile" - - // k8client "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/source" - // gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/wso2/apk/adapter/pkg/logging" cache "github.com/wso2/apk/common-controller/internal/cache" @@ -54,8 +38,6 @@ import ( loggers "github.com/wso2/apk/common-controller/internal/loggers" "github.com/wso2/apk/common-controller/internal/utils" xds "github.com/wso2/apk/common-controller/internal/xds" - // dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" - // dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" "github.com/wso2/apk/common-go-libs/constants" ) @@ -73,12 +55,6 @@ func NewAIRatelimitController(mgr manager.Manager, ratelimitStore *cache.Ratelim ods: ratelimitStore, } - // ctx := context.Background() - // if err := addIndexes(ctx, mgr); err != nil { - // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2612, logging.BLOCKER, "Error adding indexes: %v", err)) - // return err - // } - c, err := controller.New(constants.AIRatelimitController, mgr, controller.Options{Reconciler: aiRateLimitPolicyReconciler}) if err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2663, logging.BLOCKER, @@ -89,20 +65,6 @@ func NewAIRatelimitController(mgr manager.Manager, ratelimitStore *cache.Ratelim conf := config.ReadConfigs() predicates := []predicate.Predicate{predicate.NewPredicateFuncs(utils.FilterByNamespaces(conf.CommonController.Operator.Namespaces))} - // if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha2.API{}), - // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForAPI), predicates...); err != nil { - // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2611, logging.BLOCKER, - // "Error watching API resources: %v", err)) - // return err - // } - - // if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.HTTPRoute{}), - // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForHTTPRoute), predicates...); err != nil { - // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2613, logging.BLOCKER, - // "Error watching HTTPRoute resources: %v", err)) - // return err - // } - if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), &handler.EnqueueRequestForObject{}, predicates...); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2639, logging.BLOCKER, "Error watching Ratelimit resources: %v", err.Error())) @@ -153,7 +115,6 @@ func (r *AIRateLimitPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Re xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) } } - loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile..*****.") return ctrl.Result{}, nil } diff --git a/common-controller/internal/xds/ratelimiter_cache.go b/common-controller/internal/xds/ratelimiter_cache.go index 53827e7230..035c6c91f4 100644 --- a/common-controller/internal/xds/ratelimiter_cache.go +++ b/common-controller/internal/xds/ratelimiter_cache.go @@ -335,10 +335,8 @@ func (r *rateLimitPolicyCache) AddCustomRateLimitPolicies(customRateLimitPolicy // ProcessSubscriptionBasedAIRatelimitPolicySpecsAndUpdateCache process the specs and update the cache func (r *rateLimitPolicyCache) ProcessSubscriptionBasedAIRatelimitPolicySpecsAndUpdateCache(subscriptionEnabledAIRatelimitPolicies map[types.NamespacedName]struct{}, aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) - loggers.LoggerAPKOperator.Infof("222222") for namespacedNameRl := range subscriptionEnabledAIRatelimitPolicies { if airl, exists := aiRatelimitPolicySpecs[namespacedNameRl]; exists { - loggers.LoggerAPKOperator.Infof("----- %s %s %s", DescriptorKeyForSubscriptionBasedAIRequestTokenCount, prepareSubscriptionBasedAIRatelimitIdentifier(airl.Override.Organization, namespacedNameRl), DescriptorKeyForSubscription) // Add descriptor for RequestTokenCount aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ Key: DescriptorKeyForSubscriptionBasedAIRequestTokenCount, @@ -403,10 +401,7 @@ func (r *rateLimitPolicyCache) ProcessSubscriptionBasedAIRatelimitPolicySpecsAnd // ProcessAIratelimitPolicySpecsAndUpdateCache process the specs and update the cache func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRateLimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) - loggers.LoggerAPKOperator.Infof("222222") for namespacedName, spec := range aiRateLimitPolicySpecs { - logger.Infof("Adding : %s, %s", DescriptorKeyForAIRequestCount, prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec)) - logger.Infof("For airl: %s", namespacedName) // Add descriptor for RequestTokenCount aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ Key: DescriptorKeyForAIRequestTokenCount, diff --git a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle index 5a7bc193a7..94b5a29298 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle +++ b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle @@ -87,6 +87,7 @@ dependencies { implementation libs.gson implementation libs.ua.parser implementation libs.commons.lang3 + implementation libs.commons.compress implementation libs.openfeign.feign.gson implementation libs.openfeign.feign.slf4j diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java index b961743b37..060a441b07 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExtAuthService.java @@ -270,7 +270,6 @@ private String constructQueryParamString(boolean removeAllQueryParams, String re * @param value */ private void addMetadata(Struct.Builder structBuilder, String key, String value) { - System.out.println("Key: " + key + " value: " + value); structBuilder.putFields(key, Value.newBuilder().setStringValue(value).build()); } @@ -309,6 +308,4 @@ private CheckResponse buildReadyCheckResponse(CheckResponse.Builder responseBuil .setDeniedResponse(deniedResponsePreparer.build()) .build(); } - - } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index 4d776ad9b3..258c655462 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -22,12 +22,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.Struct; import com.google.protobuf.Value; +import io.envoyproxy.envoy.config.core.v3.HeaderValue; import io.envoyproxy.envoy.service.ext_proc.v3.BodyMutation; import io.envoyproxy.envoy.service.ext_proc.v3.BodyResponse; import io.envoyproxy.envoy.service.ext_proc.v3.CommonResponse; import io.envoyproxy.envoy.service.ext_proc.v3.ExternalProcessorGrpc; import io.envoyproxy.envoy.service.ext_proc.v3.HeaderMutation; import io.envoyproxy.envoy.service.ext_proc.v3.HeadersResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.HttpHeaders; import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingRequest; import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingResponse; import io.grpc.stub.StreamObserver; @@ -73,26 +75,65 @@ public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProc public StreamObserver process( final StreamObserver responseObserver) { FilterMetadata filterMetadata = new FilterMetadata(); - System.out.println("process ...."); return new StreamObserver() { @Override public void onNext(ProcessingRequest request) { ProcessingRequest.RequestCase r = request.getRequestCase(); switch (r) { + case RESPONSE_HEADERS: + if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ + Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); + FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); + filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; + filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; + } + executorService.submit(() -> { + Struct filterMetadataFromAuthZ = request.getMetadataContext().getFilterMetadataOrDefault("envoy.filters.http.ext_authz", null); + if (filterMetadataFromAuthZ != null) { + String extractTokenFrom = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_EXTRACT_TOKEN_FROM).getStringValue(); + String promptTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_PROMPT_TOKEN_ID).getStringValue(); + String completionTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_COMPLETION_TOKEN_ID).getStringValue(); + String totalTokenID = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_TOTAL_TOKEN_ID).getStringValue(); + + Usage usage = extractUsageFromHeaders(request.getResponseHeaders(), completionTokenID, promptTokenID, totalTokenID); + if (usage == null) { + logger.error("Usage details not found.."); + responseObserver.onCompleted(); + return; + } + List configs = new ArrayList<>(); + if (filterMetadata.enableBackendBasedAIRatelimit) { + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens() - 1)); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens() - 1)); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens() - 1)); + } + if (request.hasMetadataContext()) { + if (filterMetadataFromAuthZ != null) { + if (filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY) != null && filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION) != null) { + String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); + String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens() - 1))); + } + } + } + ratelimitClient.shouldRatelimit(configs); + } + }); + responseObserver.onCompleted(); case RESPONSE_BODY: if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; - filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; } - System.out.println("In the response flow metadata descirtor:" + filterMetadata.backendBasedAIRatelimitDescriptorValue); if (request.hasResponseBody()) { final byte[] bodyFromResponse = request.getResponseBody().getBody().toByteArray(); executorService.submit(() -> { - String body = null; + String body; try { body = decompress(bodyFromResponse); } catch (Exception e) { @@ -118,16 +159,14 @@ public void onNext(ProcessingRequest request) { configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens() - 1)); configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens() - 1)); } - if (filterMetadata.enableSubscriptionBasedAIRatelimit) { - if (request.hasMetadataContext()) { - if (filterMetadataFromAuthZ != null) { - if (filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY) != null && filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION) != null) { - String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); - String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens() - 1))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens() - 1))); - configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens() - 1))); - } + if (request.hasMetadataContext()) { + if (filterMetadataFromAuthZ != null) { + if (filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY) != null && filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION) != null) { + String orgAndAIRLPolicyValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_ORGANIZATION_AND_AIRL_POLICY).getStringValue(); + String aiRLSubsValue = filterMetadataFromAuthZ.getFieldsMap().get(DYNAMIC_METADATA_KEY_FOR_SUBSCRIPTION).getStringValue(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_REQUEST_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getPrompt_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_RESPONSE_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getCompletion_tokens() - 1))); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_SUBSCRIPTION_BASED_AI_TOTAL_TOKEN_COUNT, orgAndAIRLPolicyValue, new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_SUBSCRIPTION, aiRLSubsValue, usage.getTotal_tokens() - 1))); } } } @@ -136,7 +175,6 @@ public void onNext(ProcessingRequest request) { }); responseObserver.onCompleted(); } else { - System.out.println("Request does not have response body"); responseObserver.onCompleted(); } @@ -145,7 +183,7 @@ public void onNext(ProcessingRequest request) { @Override public void onError(Throwable err) { - System.out.println("on error ...."+ err.getLocalizedMessage() + " " + err.getMessage() + " " + err.toString()+ " ****"); + logger.error("Error initiated from envoy in the external processing session. Error: " + err); } @Override @@ -180,13 +218,11 @@ protected HeadersResponse prepareHeadersResponse() { // The FilterMetadata class as per your request private static class FilterMetadata { - boolean enableSubscriptionBasedAIRatelimit; boolean enableBackendBasedAIRatelimit; String backendBasedAIRatelimitDescriptorValue; @Override public String toString() { return "FilterMetadata{" + - "enableSubscriptionBasedAIRatelimit=" + enableSubscriptionBasedAIRatelimit + ", enableBackendBasedAIRatelimit=" + enableBackendBasedAIRatelimit + ", backendBasedAIRatelimitDescriptorValue='" + backendBasedAIRatelimitDescriptorValue + '\'' + '}'; @@ -199,12 +235,10 @@ public static FilterMetadata convertStringToFilterMetadata(String input) { // Regex patterns to extract specific fields String backendValuePattern = "key: \"BackendBasedAIRatelimitDescriptorValue\".*?string_value: \"(.*?)\""; String enableBackendPattern = "key: \"EnableBackendBasedAIRatelimit\".*?string_value: \"(.*?)\""; - String enableSubscriptionPattern = "key: \"EnableSubscriptionBasedAIRatelimit\".*?string_value: \"(.*?)\""; // Extract and assign to the FilterMetadata object metadata.backendBasedAIRatelimitDescriptorValue = extractValue(input, backendValuePattern); metadata.enableBackendBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableBackendPattern)); - metadata.enableSubscriptionBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableSubscriptionPattern)); return metadata; } @@ -224,6 +258,27 @@ public static String sanitize(String input) { return input.replaceAll("[\\t\\n\\r]+", " ").trim(); } + private static Usage extractUsageFromHeaders(HttpHeaders headers, String completionTokenPath, String promptTokenPath, String totalTokenPath) { + try { + Usage usage = new Usage(); + for (HeaderValue headerValue : headers.getHeaders().getHeadersList()) { + if (headerValue.getKey().equals(completionTokenPath)) { + usage.completion_tokens = Integer.parseInt(headerValue.getValue()); + } + if (headerValue.getKey().equals(promptTokenPath)) { + usage.prompt_tokens = Integer.parseInt(headerValue.getValue()); + } + if (headerValue.getKey().equals(totalTokenPath)) { + usage.total_tokens = Integer.parseInt(headerValue.getValue()); + } + } + return usage; + } catch (Exception e) { + logger.error("Error occured while getting yusage info from headers" + e); + return null; + } + } + private static Usage extractUsageFromBody(String body, String completionTokenPath, String promptTokenPath, String totalTokenPath) { body = sanitize(body); ObjectMapper mapper = new ObjectMapper(); @@ -284,7 +339,7 @@ private static Usage extractUsageFromBody(String body, String completionTokenPat return usage; } catch (Exception e) { - System.out.println(String.format("Unexpected error while extracting usage from the body: %s", body) + " \n" + e); + logger.error(String.format("Unexpected error while extracting usage from the body: %s", body) + " \n" + e); return null; } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java index f62d293e1c..e0b341cd13 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -8,17 +8,20 @@ import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.wso2.apk.enforcer.config.ConfigHolder; +import org.wso2.apk.enforcer.grpc.ExternalProcessorService; import java.io.File; import java.nio.file.Paths; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import javax.net.ssl.SSLException; public class RatelimitClient { + private static final Logger logger = LogManager.getLogger(RatelimitClient.class); RateLimitServiceGrpc.RateLimitServiceBlockingStub stub; + public RatelimitClient(){ File certFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPublicKeyPath()).toFile(); File keyFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPrivateKeyPath()).toFile(); @@ -30,7 +33,7 @@ public RatelimitClient(){ .keyManager(certFile, keyFile) .build(); } catch (SSLException e) { - System.out.println("Error while generating SSL Context."+ e); + logger.error("Error while generating SSL Context."+ e); } String rlHost = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterHost(); int port = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterPort(); @@ -59,7 +62,6 @@ public void shouldRatelimit(List configs) { .setHitsAddend(hitsAddend) .build(); RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); - System.out.println(rateLimitResponse.getOverallCode()); } } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java index daf2aa091a..c6bfc3fdae 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/KeyValidator.java @@ -263,8 +263,6 @@ public static APIKeyValidationInfoDTO validateSubscription(String apiUuid, Strin return infoDTO; } - - private static void validate(APIKeyValidationInfoDTO infoDTO, Application app, Subscription sub) { // Validate subscription status diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java index 0eb2a90ef1..eba64e2c25 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/security/jwt/APIKeyAuthenticator.java @@ -222,8 +222,8 @@ private AuthenticationContext processAPIKey(RequestContext requestContext, Strin .getMandateSubscriptionValidation(); if (!requestContext.getMatchedAPI().isSystemAPI() && (isGatewayLevelSubscriptionValidationEnabled || requestContext.getMatchedAPI() .isSubscriptionValidation())) { - validationInfoDto = KeyValidator.validateSubscription(apiUuid, apiContext, - requestContext.getMatchedAPI(), payload); +// validationInfoDto = KeyValidator.validateSubscription(apiUuid, apiContext, +// requestContext.getMatchedAPI(), payload); log.debug("Validating subscription for API Key using JWT claims against invoked API info." + " context: {} version: {}", apiContext, apiVersion); validationInfoDto = getAPIKeyValidationDTO(requestContext, payload); diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java index d849dc648a..38ac92af25 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java @@ -159,7 +159,6 @@ private static Server initServer() throws SSLException { EnforcerWorkerPool enforcerWorkerPool = new EnforcerWorkerPool(threadPoolConfig.getCoreSize(), threadPoolConfig.getMaxSize(), threadPoolConfig.getKeepAliveTime(), threadPoolConfig.getQueueSize(), Constants.EXTERNAL_AUTHZ_THREAD_GROUP, Constants.EXTERNAL_AUTHZ_THREAD_ID); - System.out.println("test"); return NettyServerBuilder.forPort(authServerConfig.getPort()) .keepAliveTime(authServerConfig.getKeepAliveTime(), TimeUnit.SECONDS).bossEventLoopGroup(bossGroup) .workerEventLoopGroup(workerGroup) diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java index b8a31efaad..42cb28a935 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/subscription/SubscriptionDto.java @@ -63,4 +63,4 @@ public void setRatelimitTier(String ratelimitTier) { this.ratelimitTier = ratelimitTier; } -} \ No newline at end of file +} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index bf6217a4ff..50d9bc91f8 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -100,9 +100,9 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} - value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers }} {{- $defaultPublisherSecretName := "" }} From 6fac10901c3d2a045aaa0e4d68772198f610e2e6 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 13:34:15 +0530 Subject: [PATCH 15/24] Add integration test for AIRL --- .../envoyconf/routes_with_clusters.go | 2 +- .../grpc/ExternalProcessorService.java | 29 ++- test/cucumber-tests/CRs/llm-test-header.yaml | 163 ++++++++++++++ test/cucumber-tests/CRs/llm-test-subs.yaml | 200 ++++++++++++++++++ test/cucumber-tests/CRs/llm-test.yaml | 163 ++++++++++++++ .../integration/api/JWTGeneratorSteps.java | 9 +- .../api/APIBackendBasedAIRatelimit.feature | 86 ++++++++ .../APISubscriptionBasedAIRatelimit.feature | 52 +++++ 8 files changed, 698 insertions(+), 6 deletions(-) create mode 100644 test/cucumber-tests/CRs/llm-test-header.yaml create mode 100644 test/cucumber-tests/CRs/llm-test-subs.yaml create mode 100644 test/cucumber-tests/CRs/llm-test.yaml create mode 100644 test/cucumber-tests/src/test/resources/tests/api/APIBackendBasedAIRatelimit.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 772031f383..a11f01fff2 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -885,7 +885,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error Overrides: &extProcessorv3.ExtProcOverrides{ ProcessingMode: &extProcessorv3.ProcessingMode{ RequestHeaderMode: extProcessorv3.ProcessingMode_SKIP, - ResponseHeaderMode: extProcessorv3.ProcessingMode_SKIP, + ResponseHeaderMode: extProcessorv3.ProcessingMode_SEND, ResponseBodyMode: extProcessorv3.ProcessingMode_NONE, }, }, diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java index 258c655462..0a2dbaf343 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -261,18 +261,39 @@ public static String sanitize(String input) { private static Usage extractUsageFromHeaders(HttpHeaders headers, String completionTokenPath, String promptTokenPath, String totalTokenPath) { try { Usage usage = new Usage(); + boolean completionTokenExtracted = false; + boolean promptTokenExtracted = false; + boolean totalTokenExtracted = false; for (HeaderValue headerValue : headers.getHeaders().getHeadersList()) { if (headerValue.getKey().equals(completionTokenPath)) { - usage.completion_tokens = Integer.parseInt(headerValue.getValue()); + completionTokenExtracted = true; + String value = headerValue.getValue(); + if (value.isEmpty()) { + value = headerValue.getRawValue().toString(StandardCharsets.UTF_8); + } + usage.completion_tokens = Integer.parseInt(value); } if (headerValue.getKey().equals(promptTokenPath)) { - usage.prompt_tokens = Integer.parseInt(headerValue.getValue()); + promptTokenExtracted = true;completionTokenExtracted = true; + String value = headerValue.getValue(); + if (value.isEmpty()) { + value = headerValue.getRawValue().toString(StandardCharsets.UTF_8); + } + usage.prompt_tokens = Integer.parseInt(value); } if (headerValue.getKey().equals(totalTokenPath)) { - usage.total_tokens = Integer.parseInt(headerValue.getValue()); + totalTokenExtracted = true;completionTokenExtracted = true; + String value = headerValue.getValue(); + if (value.isEmpty()) { + value = headerValue.getRawValue().toString(StandardCharsets.UTF_8); + } + usage.total_tokens = Integer.parseInt(value); } } - return usage; + if (completionTokenExtracted && promptTokenExtracted && totalTokenExtracted) { + return usage; + } + return null; } catch (Exception e) { logger.error("Error occured while getting yusage info from headers" + e); return null; diff --git a/test/cucumber-tests/CRs/llm-test-header.yaml b/test/cucumber-tests/CRs/llm-test-header.yaml new file mode 100644 index 0000000000..95d3c3e3d5 --- /dev/null +++ b/test/cucumber-tests/CRs/llm-test-header.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment-header + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app-header + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app-header + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container-header + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service-header + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app-header + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api-header + namespace: apk-integration-test +spec: + apiName: llm-api-header + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api-header/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route-header + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route-header + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend-header +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend-header + namespace: apk-integration-test +spec: + services: + - host: llm-service-header + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl-header + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute + targetRef: + kind: Backend + name: llm-backend-header + group: gateway.networking.k8s.io +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy-header + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider-header" + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api-header +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider-header + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Header" + value: "model" + rateLimitFields: + promptTokens: + in: "Header" + value: "prompt_tokens" + completionToken: + in: "Header" + value: "completion_tokens" + totalToken: + in: "Header" + value: "total_tokens" diff --git a/test/cucumber-tests/CRs/llm-test-subs.yaml b/test/cucumber-tests/CRs/llm-test-subs.yaml new file mode 100644 index 0000000000..e5a3c6696e --- /dev/null +++ b/test/cucumber-tests/CRs/llm-test-subs.yaml @@ -0,0 +1,200 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment-subs + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app-subs + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app-subs + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container-subs + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service-subs + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app-subs + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api-subs + namespace: apk-integration-test +spec: + apiName: llm-api-subs + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api-subs/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route-subs + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route-subs + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend-subs +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend-subs + namespace: apk-integration-test +spec: + services: + - host: llm-service-subs + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl-subs + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy-subs + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider-subs" + subscriptionValidation: true + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api-subs +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider-subs + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Body" + value: "model" + rateLimitFields: + promptTokens: + in: "Body" + value: "prompt_tokens" + completionToken: + in: "Body" + value: "completion_tokens" + totalToken: + in: "Body" + value: "total_tokens" +--- +apiVersion: cp.wso2.com/v1alpha2 +kind: Subscription +metadata: + name: llm-subs + namespace: apk-integration-test +spec: + organization: "default" + subscriptionStatus: "ACTIVE" + api: + name: "llm-api-subs" + version: "v1.0.0" + ratelimitRef: + name : llm-backend-rl-subs + level: application +--- +apiVersion: cp.wso2.com/v1alpha2 +kind: ApplicationMapping +metadata: + name: llm-app-map + namespace: apk-integration-test +spec: + applicationRef: llm-app + subscriptionRef: llm-subs +--- +kind: Application +apiVersion: cp.wso2.com/v1alpha2 +metadata: + name: llm-app + namespace : apk-integration-test +spec: + name: sample-app + owner: admin + organization: default + securitySchemes: + oauth2: + environments: + - envId: Default + appId: 45f1c5c8-a92e-11ed-afa1-0242ac120005 + keyType: PRODUCTION \ No newline at end of file diff --git a/test/cucumber-tests/CRs/llm-test.yaml b/test/cucumber-tests/CRs/llm-test.yaml new file mode 100644 index 0000000000..3a2b0902fc --- /dev/null +++ b/test/cucumber-tests/CRs/llm-test.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api + namespace: apk-integration-test +spec: + apiName: llm-api + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend + namespace: apk-integration-test +spec: + services: + - host: llm-service + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute + targetRef: + kind: Backend + name: llm-backend + group: gateway.networking.k8s.io +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider" + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Body" + value: "model" + rateLimitFields: + promptTokens: + in: "Body" + value: "usage.prompt_tokens" + completionToken: + in: "Body" + value: "usage.completion_tokens" + totalToken: + in: "Body" + value: "usage.total_tokens" \ No newline at end of file diff --git a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/JWTGeneratorSteps.java b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/JWTGeneratorSteps.java index 9b3d940288..4aa80b4e06 100644 --- a/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/JWTGeneratorSteps.java +++ b/test/cucumber-tests/src/test/java/org/wso2/apk/integration/api/JWTGeneratorSteps.java @@ -70,7 +70,7 @@ public void generateTokenFromIdp1WithConsumerKey(String kid,String consumerKey) JWTClaimsSet claimsSet = new JWTClaimsSet.Builder() .subject("alice") .issuer("https://idp1.com") - .expirationTime(new Date(new Date().getTime() + 60 * 1000)) + .expirationTime(new Date(new Date().getTime() + 60 * 60 * 24 * 1000)) .jwtID(UUID.randomUUID().toString()) .claim("azp", consumerKey) .claim("scope", Constants.API_CREATE_SCOPE) @@ -83,6 +83,13 @@ public void generateTokenFromIdp1WithConsumerKey(String kid,String consumerKey) sharedContext.addStoreValue("idp-1-"+consumerKey+"-token", jwtToken); } + public static void main(String[] args) throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException, JOSEException { + SharedContext sharedContext1 = new SharedContext(); + String consumerKey = "45f1c5c8-a92e-11ed-afa1-0242ac120005"; + new JWTGeneratorSteps(sharedContext1).generateTokenFromIdp1WithConsumerKey("123-456", consumerKey); + System.out.println(sharedContext1.getStoreValue("idp-1-"+consumerKey+"-token")); + } + @And("I have a valid token for organization {string}") public void generateTokenFromIdp1WithOrganization(String organization) throws IOException, CertificateException, diff --git a/test/cucumber-tests/src/test/resources/tests/api/APIBackendBasedAIRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/APIBackendBasedAIRatelimit.feature new file mode 100644 index 0000000000..2ad7bebe3b --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/APIBackendBasedAIRatelimit.feature @@ -0,0 +1,86 @@ +Feature: API backend based AI ratelimit Feature + Scenario: backend based AI ratelimit token detail comes in the body. + Given The system is ready + And I have a valid subscription + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4999 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4699 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body&prompt_tokens=40000" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4399 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body&prompt_tokens=40000" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body&completion_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body&total_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 429 + Scenario: backend based AI ratelimit token detail comes in the header. + Given The system is ready + And I have a valid subscription + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4999 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4699 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header&prompt_tokens=40000" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4399 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header&prompt_tokens=40000" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header&completion_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header&total_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-header/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 429 + Scenario: backend based AI ratelimit token detail comes in the header but a body configured api checked. + Given The system is ready + And I have a valid subscription + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4999 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4998 | \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature new file mode 100644 index 0000000000..eacb321d30 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature @@ -0,0 +1,52 @@ +Feature: API subscription based AI ratelimit Feature + Scenario: subscription based AI ratelimit token detail comes in the body. + Given The system is ready + And I have a valid subscription + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4999 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4699 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body&prompt_tokens=40000" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4399 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body&prompt_tokens=40000" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body&completion_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 429 + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body&total_tokens=40000" with body "" + Then the response status code should be 200 + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" + Then the response status code should be 429 + + Scenario: subs based AI ratelimit token detail comes in the header but a body configured api checked. + Given The system is ready + And I have a valid subscription + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute strictly + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4999 | + And I wait for 3 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=header" with body "" + Then the response status code should be 200 + And the response headers should contain + | x-ratelimit-remaining | 4998 | \ No newline at end of file From 3d46fce8c0e325d023cbf1be486b5ad0fa1585b8 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 14:18:18 +0530 Subject: [PATCH 16/24] Fix test artifacts --- .../oasparser/envoyconf/routes_with_clusters.go | 16 ---------------- test/cucumber-tests/CRs/llm-test-subs.yaml | 6 +++--- .../api/APISubscriptionBasedAIRatelimit.feature | 6 ++++-- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index a11f01fff2..157b51a770 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -1120,11 +1120,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableBackendBasedAIRatelimit() && params.isAiAPI, resource.GetBackendBasedAIRatelimitDescriptorValue()) requestHeadersToRemove := make([]string,0) - if params.isAiAPI { - requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") - } else { - requestHeadersToRemove = nil - } // Create route1 for current method. // Do not add policies to route config. Send via enforcer route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, @@ -1138,9 +1133,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action2.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } configToSkipEnforcer := generateFilterConfigToSkipEnforcer() - if params.isAiAPI { - requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") - } route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, metaData, decorator, configToSkipEnforcer, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) @@ -1161,9 +1153,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else if requestRedirectAction == nil { action.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } - if params.isAiAPI { - requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") - } route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route) @@ -1182,11 +1171,6 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) requestHeadersToRemove := make([]string,0) - if params.isAiAPI { - requestHeadersToRemove = append(requestHeadersToRemove, "Accept-Encoding", "accept-encoding") - } else { - requestHeadersToRemove = nil - } route := generateRouteConfig(xWso2Basepath, match, action, nil, metaData, decorator, perRouteFilterConfigs, nil, requestHeadersToRemove, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) diff --git a/test/cucumber-tests/CRs/llm-test-subs.yaml b/test/cucumber-tests/CRs/llm-test-subs.yaml index e5a3c6696e..8856f901bc 100644 --- a/test/cucumber-tests/CRs/llm-test-subs.yaml +++ b/test/cucumber-tests/CRs/llm-test-subs.yaml @@ -151,13 +151,13 @@ spec: rateLimitFields: promptTokens: in: "Body" - value: "prompt_tokens" + value: "usage.prompt_tokens" completionToken: in: "Body" - value: "completion_tokens" + value: "usage.completion_tokens" totalToken: in: "Body" - value: "total_tokens" + value: "usage.total_tokens" --- apiVersion: cp.wso2.com/v1alpha2 kind: Subscription diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature index eacb321d30..7f908508cd 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature @@ -2,8 +2,9 @@ Feature: API subscription based AI ratelimit Feature Scenario: subscription based AI ratelimit token detail comes in the body. Given The system is ready And I have a valid subscription + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" Then I set headers - |Authorization|bearer ${accessToken}| + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| And I wait for next minute strictly And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" Then the response status code should be 200 @@ -38,8 +39,9 @@ Feature: API subscription based AI ratelimit Feature Scenario: subs based AI ratelimit token detail comes in the header but a body configured api checked. Given The system is ready And I have a valid subscription + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" Then I set headers - |Authorization|bearer ${accessToken}| + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| And I wait for next minute strictly And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=header" with body "" Then the response status code should be 200 From 025a367b7a2bec993ba66aaca8d4186fc88a83af Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 14:55:30 +0530 Subject: [PATCH 17/24] Fix formats --- test/cucumber-tests/CRs/llm-test-header.yaml | 4 ++-- test/cucumber-tests/CRs/llm-test-subs.yaml | 2 +- test/cucumber-tests/CRs/llm-test.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/cucumber-tests/CRs/llm-test-header.yaml b/test/cucumber-tests/CRs/llm-test-header.yaml index 95d3c3e3d5..7fd2bff518 100644 --- a/test/cucumber-tests/CRs/llm-test-header.yaml +++ b/test/cucumber-tests/CRs/llm-test-header.yaml @@ -119,7 +119,7 @@ spec: totalTokenCount: 15000 requestCount: requestsPerUnit: 6000 - unit: Minute + unit: Minute targetRef: kind: Backend name: llm-backend-header @@ -152,7 +152,7 @@ spec: in: "Header" value: "model" rateLimitFields: - promptTokens: + promptTokens: in: "Header" value: "prompt_tokens" completionToken: diff --git a/test/cucumber-tests/CRs/llm-test-subs.yaml b/test/cucumber-tests/CRs/llm-test-subs.yaml index 8856f901bc..b33c66ca27 100644 --- a/test/cucumber-tests/CRs/llm-test-subs.yaml +++ b/test/cucumber-tests/CRs/llm-test-subs.yaml @@ -149,7 +149,7 @@ spec: in: "Body" value: "model" rateLimitFields: - promptTokens: + promptTokens: in: "Body" value: "usage.prompt_tokens" completionToken: diff --git a/test/cucumber-tests/CRs/llm-test.yaml b/test/cucumber-tests/CRs/llm-test.yaml index 3a2b0902fc..d45cda6815 100644 --- a/test/cucumber-tests/CRs/llm-test.yaml +++ b/test/cucumber-tests/CRs/llm-test.yaml @@ -119,7 +119,7 @@ spec: totalTokenCount: 15000 requestCount: requestsPerUnit: 6000 - unit: Minute + unit: Minute targetRef: kind: Backend name: llm-backend @@ -152,7 +152,7 @@ spec: in: "Body" value: "model" rateLimitFields: - promptTokens: + promptTokens: in: "Body" value: "usage.prompt_tokens" completionToken: From b86be74ef165a5cec834ae61e59145f097858f4a Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 14:58:50 +0530 Subject: [PATCH 18/24] Add tests --- .../tests/api/APIDefinitionEndpoint.feature | 84 ++++ .../tests/api/APISubscription.feature | 72 ++++ .../resources/tests/api/BackendRetry.feature | 35 ++ .../tests/api/BackendTimeout.feature | 48 +++ .../resources/tests/api/BasicAuth.feature | 43 +++ .../BasicDeploymentAndApiInvocation.feature | 138 +++++++ .../src/test/resources/tests/api/CORS.feature | 60 +++ .../tests/api/CircuitBreaker.feature | 36 ++ .../api/CircuitBreakerEnvoyConfigDump.feature | 26 ++ .../api/CircuitBreakerMaxRequest.feature | 47 +++ .../tests/api/CustomRatelimit.feature | 118 ++++++ .../DifferentEndpointResourceLevel.feature | 28 ++ .../api/DifferentSandProdEndpoint.feature | 34 ++ .../test/resources/tests/api/Endpoint.feature | 37 ++ .../tests/api/GlobalInterceptor.feature | 25 ++ .../test/resources/tests/api/GraphQL.feature | 237 ++++++++++++ .../tests/api/HeaderModifier.feature | 84 ++++ .../resources/tests/api/Interceptor.feature | 119 ++++++ .../src/test/resources/tests/api/JWT.feature | 126 ++++++ .../tests/api/MTLSwithOAuth2Mandatory.feature | 358 ++++++++++++++++++ .../tests/api/MTLSwithOAuth2Optional.feature | 180 +++++++++ .../tests/api/MultiEnvironment.feature | 117 ++++++ .../tests/api/OrganizationBasedAPIS.feature | 47 +++ .../resources/tests/api/RequestMirror.feature | 21 + .../tests/api/RequestRedirect.feature | 51 +++ .../resources/tests/api/RevokedToken.feature | 35 ++ .../tests/api/SemanticVersioning.feature | 118 ++++++ .../tests/api/SimpleRateLimit.feature | 76 ++++ .../resources/tests/api/deployment.feature | 44 +++ .../tests/api/resourceInterceptor.feature | 34 ++ 30 files changed, 2478 insertions(+) create mode 100644 test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/CORS.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/JWT.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/deployment.feature create mode 100644 test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature diff --git a/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature new file mode 100644 index 0000000000..b96dff9607 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/APIDefinitionEndpoint.feature @@ -0,0 +1,84 @@ +Feature: API Definition Endpoint + Scenario: Testing default API definition endpoint + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/api_definition_default.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" + And I eventually receive 200 response code, not accepting + |429| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/api-definition" with body "" + And I eventually receive 200 response code, not accepting + |429| + + Scenario: Testing custom API definition endpoint + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/api_definition_custom.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/3.14/docs" with body "" + And I eventually receive 200 response code, not accepting + |429| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/docs" with body "" + And I eventually receive 200 response code, not accepting + |429| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/api-definition" with body "" + And I eventually receive 404 response code, not accepting + |429| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-custom/3.14/api-definition" with body "" + And I eventually receive 404 response code, not accepting + |429| + + Scenario: Testing a deleted production endpoint + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/api_definition_default_without_production.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I wait for 1 minute + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" + Then the response status code should be 404 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-definition-default/api-definition" with body "" + Then the response status code should be 404 + + Scenario: Testing a deleted production endpoint + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/api_definition_default.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + When I use the APK Conf file "artifacts/apk-confs/api_definition_default_without_sandbox.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I wait for 1 minute + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-definition-default/3.14/api-definition" with body "" + Then the response status code should be 404 + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-definition-default/api-definition" with body "" + Then the response status code should be 404 + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | custom-api-definition-endpoint-test | 202 | + | default-api-definition-endpoint-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature new file mode 100644 index 0000000000..36304a6ab4 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/APISubscription.feature @@ -0,0 +1,72 @@ +Feature: API Subscription Feature + Scenario: testing api subscriptions. + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/subscription-api.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 403 response code, not accepting + |429| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + Then the response status code should be 403 + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + Then the response status code should be 403 + + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120006" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120006-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 403 response code, not accepting + |200| + |429| + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120007" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120007-token}| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 403 response code, not accepting + |200| + |429| + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120008" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120008-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 403 response code, not accepting + |200| + |401| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + And I eventually receive 403 response code, not accepting + |200| + |429| + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120009" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120009-token}| + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + Then the response status code should be 403 + And I send "GET" request to "https://default.gw.wso2.com:9095/subscription-api/1.0.0/endpoint1" with body "" + Then the response status code should be 403 + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | subscription-api | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature b/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature new file mode 100644 index 0000000000..a9451fa629 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/BackendRetry.feature @@ -0,0 +1,35 @@ +Feature: BackendRetry + Scenario: Testing backend retry + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/backend_retry_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" + And I eventually receive 200 response code, not accepting + |429| + And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/set-retry-count" with body "{\"count\": 4}" + Then the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/reset" with body "" + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" + Then the response status code should be 500 + And I send "POST" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/set-retry-count" with body "{\"count\": 2}" + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/backend-retry/3.14/retry" with body "" + Then the response status code should be 200 + + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | backend-retry-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature b/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature new file mode 100644 index 0000000000..1c3279a9a0 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/BackendTimeout.feature @@ -0,0 +1,48 @@ +Feature: BackendTimeout + Scenario: Testing backend timeout + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/backend_timeout_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/2" with body "" + And I eventually receive 504 response code, not accepting + |429| + |200| + And the response body should contain "timeout" + And I send "GET" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/0" with body "" + And I eventually receive 200 response code, not accepting + |429| + |500| + And I send "POST" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/3" with body "" + And I eventually receive 504 response code, not accepting + |429| + |200| + And the response body should contain "timeout" + And I send "POST" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/1" with body "" + And I eventually receive 200 response code, not accepting + |429| + |500| + And I send "PUT" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/4" with body "" + And I eventually receive 504 response code, not accepting + |429| + |200| + And the response body should contain "timeout" + And I send "PUT" request to "https://default.gw.wso2.com:9095/backend-timeout/3.14/delay/2" with body "" + And I eventually receive 200 response code, not accepting + |429| + |500| + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | backend-timeout-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature b/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature new file mode 100644 index 0000000000..603680b21f --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/BasicAuth.feature @@ -0,0 +1,43 @@ +Feature: Basic auth + Scenario: Testing API level and resource level basic auth header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/basic_auth_conf.yaml" + And the definition file "artifacts/definitions/basic_auth_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "\"Authorization\": \"Basic YWRtaW46YWRtaW4=\"" + And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/get" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "\"Authorization\": \"Basic ZHNmZHNmc2Rmc2RmOmFkbWlu\"" + And I send "POST" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/post" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "\"Authorization\": \"Basic YWRtaW46YWRtaW4=\"" + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | basic-auth-api-test | 202 | + + Scenario: Testing undeployed API + Given The system is ready + And I have a valid subscription + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/basic-auth/3.14/employee/" with body "" + And I eventually receive 404 response code, not accepting + | 200 | + diff --git a/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature b/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature new file mode 100644 index 0000000000..8fb5f7f62d --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/BasicDeploymentAndApiInvocation.feature @@ -0,0 +1,138 @@ +Feature: API Deployment and invocation + Scenario: Deploying an API and basic http method invocations + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" + And the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/test/" with body "" + And the response status code should be 404 + And I send "PUT" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" + And the response status code should be 200 + And I send "DELETE" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" + And the response status code should be 200 + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" + And the response status code should be 401 + And I send "POST" request to "https://default.gw.wso2.com:9095/test/3.14/employee/" with body "" + And the response status code should be 401 + And I send "PUT" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" + And the response status code should be 401 + And I send "DELETE" request to "https://default.gw.wso2.com:9095/test/3.14/employee/12" with body "" + And the response status code should be 401 + + Scenario: Deploying an API with new version + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/new_version_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + When I use the APK Conf file "artifacts/apk-confs/new_version_conf2.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I wait for next minute + And I send "GET" request to "https://default.gw.wso2.com:9095/test-version/1.0/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-version/2.0/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + + Scenario: Deploying an API with default version + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/default_version_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/" with body "" + And the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/3.14/test/" with body "" + And the response status code should be 404 + And I send "PUT" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/12" with body "" + And the response status code should be 200 + And I send "DELETE" request to "https://default.gw.wso2.com:9095/test-default/3.14/employee/12" with body "" + And the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-default/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/employee/" with body "" + And the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/test-default/test/" with body "" + And the response status code should be 404 + And I send "PUT" request to "https://default.gw.wso2.com:9095/test-default/employee/12" with body "" + And the response status code should be 200 + And I send "DELETE" request to "https://default.gw.wso2.com:9095/test-default/employee/12" with body "" + And the response status code should be 200 + + Scenario: Scope Validation + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/employees_scope_test_conf.yaml" + And the definition file "artifacts/definitions/employees_scope_test_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithoutscope/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope1/" with body "" + And the response status code should be 403 + Given I have a valid subscription with scopes + | scope1 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithoutscope/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope1/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscope2/" with body "" + And I eventually receive 403 response code, not accepting + | 200 | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscopes/" with body "" + And I eventually receive 403 response code, not accepting + | 429 | + Given I have a valid subscription with scopes + | scope1 | + | scope2 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/test-scope/1.0.0/employeewithscopes/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | f7996dce4ac15e2af0f8ee14546c4f72988eddae | 202 | + | default-version-api-test | 202 | + | emp-api-test-scope | 202 | + | version-api-test | 202 | + | version-api-test2 | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CORS.feature b/test/cucumber-tests/src/test/resources/tests/api/CORS.feature new file mode 100644 index 0000000000..726fbfc236 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/CORS.feature @@ -0,0 +1,60 @@ +Feature: CORS Policy + + Scenario: Testing CORS Policy + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And I wait for next minute + And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" + And I eventually receive 204 response code, not accepting + | 429 | + And the response headers should not contain + | Access-Control-Allow-Origin | + | Access-Control-Allow-Credentials | + | Access-Control-Allow-Methods | + | Access-Control-Allow-Headers | + | Access-Control-Max-Age | + Then I set headers + | Origin | test.domain.com | + And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" + And I eventually receive 204 response code, not accepting + | 429 | + And the response headers should not contain + | Access-Control-Allow-Origin | + | Access-Control-Allow-Credentials | + | Access-Control-Allow-Methods | + | Access-Control-Allow-Headers | + | Access-Control-Max-Age | + Then I set headers + | Origin | abc.com | + And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" + And I eventually receive 204 response code, not accepting + | 429 | + And the response headers should contain + | Access-Control-Allow-Origin | abc.com | + | Access-Control-Allow-Credentials | true | + Then I set headers + | Origin | abc.com | + | Access-Control-Request-Method | GET | + And I send "OPTIONS" request to "https://default.gw.wso2.com:9095/test_cors/2.0.0/anything/" with body "" + And I eventually receive 200 response code, not accepting + | 429 | + And the response headers should contain + | Access-Control-Allow-Origin | abc.com | + | Access-Control-Allow-Credentials | true | + | Access-Control-Allow-Methods | GET, POST, PUT, DELETE | + | Access-Control-Allow-Headers | Content-Type, Authorization | + | Access-Control-Max-Age | 3600 | + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | cors-api-adff3dbc-2787-11ee-be56-0242ac120002 | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature new file mode 100644 index 0000000000..927f682b08 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreaker.feature @@ -0,0 +1,36 @@ +Feature: BackendRetry + Scenario: Testing backend retry + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/circuit_breaker_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/set-retry-count" with body "{\"count\": 500}" + And I eventually receive 200 response code, not accepting + |429| + And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/reset" with body "" + Then the response status code should be 200 + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/retry" with body "" + Then the response status code should be 500 + And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/set-retry-count" with body "{\"count\": 3}" + Then the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/circuit-breaker/3.14/reset" with body "" + Then the response status code should be 200 + + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | circuit-breaker-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature new file mode 100644 index 0000000000..39d17e6bcd --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerEnvoyConfigDump.feature @@ -0,0 +1,26 @@ +Feature: circuitBreakerMaxRequest + Scenario: Testing backend timeout + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-config-dump.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I wait for 5 seconds + And I send "GET" request to "http://default.gw.wso2.com:9000/config_dump" with body "" + Then the response status code should be 200 + And the response body should contain "\"max_connections\": 1111" + And the response body should contain "\"max_pending_requests\": 1112" + And the response body should contain "\"max_requests\": 1113" + And the response body should contain "\"max_retries\": 1114" + And the response body should contain "\"max_connection_pools\": 1115" + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | circuit-breaker-config-dump | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature new file mode 100644 index 0000000000..705430b751 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/CircuitBreakerMaxRequest.feature @@ -0,0 +1,47 @@ +Feature: circuitBreakerMaxRequest + Scenario: Testing backend timeout + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-max-request-test.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/anything/test" with body "" + And I eventually receive 200 response code, not accepting + |429| + |500| + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request/3.14/delay/10" with body "" + Then the response status code should be 503 + + When I use the APK Conf file "artifacts/apk-confs/circuit-breaker-max-request-test-v1.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/anything/test" with body "" + And I eventually receive 200 response code, not accepting + |429| + |500| + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" + And I send "GET" async request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/circuit-breaker-max-request-v1/3.14/delay/10" with body "" + Then the response status code should be 200 + + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | circuit-breaker-max-request-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature new file mode 100644 index 0000000000..5b7c0ecea3 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/CustomRatelimit.feature @@ -0,0 +1,118 @@ +Feature: Custom ratelimit + Scenario: Testing custom ratelimit + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/custom_ratelimit_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | user_id | bob | + | org_id | wso2 | + And I wait for next minute +# Request 1 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/3.14/employee" with body "" + And I eventually receive 200 response code, not accepting + | 429 | +# Request 2 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 3 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 4 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 5 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 6 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 7 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 8 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 9 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 10 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 + Then I set headers + | user_id | dummy | + | org_id | wso2 | +# Starting from Request 5 the org_id descriptor should not be counted +# Request 5 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 6 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 7 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 8 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 9 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 10 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 11 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Request 12 - for org_id descriptor + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 +# Test org_id only + And I wait for next minute strictly +# Request 1 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/3.14/employee" with body "" + Then the response status code should be 200 +# Request 2 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 3 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 4 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 5 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 6 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 7 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 8 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 9 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 10 + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 200 +# Request 11 - should be limitted + And I send "GET" request to "https://default.gw.wso2.com:9095/test-custom-ratelimit/employee" with body "" + Then the response status code should be 429 + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | custom-ratelimit-api | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature b/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature new file mode 100644 index 0000000000..cc9a7c18eb --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/DifferentEndpointResourceLevel.feature @@ -0,0 +1,28 @@ +Feature: API different endpoint resource level + Scenario: Testing different endpoint resource level + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/different_endpoint_resource_level.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-endpoint-resource-level/3.14/endpoint1" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/base1/endpoint1" + And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-endpoint-resource-level/endpoint2" with body "" + Then the response status code should be 200 + And the response body should contain "https://backend/anything/base2/endpoint2" + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | different-endpoint-resource-level-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature new file mode 100644 index 0000000000..8f8c028367 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/DifferentSandProdEndpoint.feature @@ -0,0 +1,34 @@ +Feature: API different endpoint resource level + Scenario: Testing different endpoint resource level + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/different_sand_prod_endpoint.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-sand-prod-endpoint/3.14/endpoint1" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/prodr/endpoint1" + And I send "GET" request to "https://default.gw.wso2.com:9095/test-different-sand-prod-endpoint/endpoint2" with body "" + Then the response status code should be 200 + And the response body should contain "https://backend/anything/prod/endpoint2" + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-different-sand-prod-endpoint/3.14/endpoint1" with body "" + Then the response status code should be 200 + And the response body should contain "https://backend/anything/sandr/endpoint1" + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/test-different-sand-prod-endpoint/endpoint2" with body "" + Then the response status code should be 200 + And the response body should contain "https://backend/anything/sand/endpoint2" + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | test-different-sand-prod-endpoint | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature b/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature new file mode 100644 index 0000000000..f9d8c30747 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/Endpoint.feature @@ -0,0 +1,37 @@ +Feature: Endpoint + Scenario: Testing API level and resource level endpoints + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/endpoint_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/endpoint/3.14/employee" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/employee" + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/endpoint/3.14/employee" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/test/employee" + And I send "POST" request to "https://default.gw.wso2.com:9095/endpoint/3.14/employee" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/test/employee" + And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/endpoint/3.14/employee" with body "" + And I eventually receive 200 response code, not accepting + |429| + And the response body should contain "https://backend/anything/employee" + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | endpoint-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature new file mode 100644 index 0000000000..41bff9d905 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/GlobalInterceptor.feature @@ -0,0 +1,25 @@ +Feature: API Deployment with Global Interceptor + Scenario: Deploying an API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/interceptors/globalInterceptor.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "579ba27a1e03e2fdf099d1b6745e265f2d495606" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/globalinterceptor/1.0.0/get" with body "" + Then the response status code should be 200 + And the response body should contain "\"Gw-Header\": \"GW-header-value\"" + Then the response headers contains key "gw-response-header" and value "GW-response-header-value" + Scenario Outline: Undeploy an API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | 579ba27a1e03e2fdf099d1b6745e265f2d495606 | 202 | \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature b/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature new file mode 100644 index 0000000000..ea144361a5 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/GraphQL.feature @@ -0,0 +1,237 @@ +Feature: Generating APK conf for GraphQL API + Scenario: Generating APK conf using a valid GraphQL API definition + Given The system is ready + When I use the definition file "artifacts/definitions/graphql_sample_api.graphql" in resources + And generate the APK conf file for a "GRAPHQL" API + Then the response status code should be 200 + + Scenario: Deploying APK conf using a valid GraphQL API definition without a subscription resource + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_conf_without_sub.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-without-sub" + Then the response status code should be 202 + + Scenario: Deploying GraphQL API with scopes + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_scopes.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 403 response code, not accepting + | 429 | + | 500 | + Given I have a valid subscription with scopes + | wso2 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-scopes" + Then the response status code should be 202 + + Scenario: Deploying a ratelimited GraphQL API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_rl.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + Then the response status code should be 429 + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-rl" + Then the response status code should be 202 + + Scenario: Deploying multiple versions of a GraphQL API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_3.0.0.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_4.0.0.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.0.0" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/4.0.0" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | graphql-v3 | 202 | + | graphql-v4 | 202 | + + Scenario: Deploying APK conf using a valid GraphQL API definition with mTLS mandatory and valid certificate + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-mtls" + Then the response status code should be 202 + + Scenario: Deploying APK conf using a valid GraphQL API definition with mTLS mandatory and no certificate + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 401 response code, not accepting + | 200 | + | 429 | + | 500 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-mtls" + Then the response status code should be 202 + + Scenario: Deploying APK conf using a valid GraphQL API definition with OAuth2 mandatory mTLS optional + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_mtls_optional_oauth2_mandatory.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + And I have a valid token with a client certificate "invalid-cert.txt" + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 401 response code, not accepting + | 429 | + | 500 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-mtls-optional" + Then the response status code should be 202 + + Scenario: Deploying GraphQL API with OAuth2 disabled + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_with_disabled_auth.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + And I send "POST" request to "https://default.gw.wso2.com:9095/graphql/3.14" with body "{\"query\":\"{ allHumans { name } }\"}" + And I eventually receive 200 response code, not accepting + | 429 | + | 500 | + And the response body should contain "\"name\":\"string\"" + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-auth-disabled" + Then the response status code should be 202 + + Scenario: Deploying APK conf using a valid GraphQL API definition containing a subscription resource + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/graphql/graphql_conf_with_sub.apk-conf" + And the definition file "artifacts/definitions/graphql_sample_api.graphql" + And make the API deployment request + Then the response status code should be 200 + + Scenario: Generating APK conf using an invalid GraphQL API definition + Given The system is ready + When I use the definition file "artifacts/definitions/invalid_graphql_api.graphql" in resources + And generate the APK conf file for a "GRAPHQL" API + Then the response status code should be 400 + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "graphql-with-sub" + Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature new file mode 100644 index 0000000000..75a9ee0705 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/HeaderModifier.feature @@ -0,0 +1,84 @@ +Feature: Test HTTPRoute Filter Header Modifier functionality + Scenario: Test request and response header modification functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/header-modifier-filter.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-header-modifier-filters" + Then the response status code should be 202 + + Scenario: Test request and response header modification functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-header.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "POST" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "PUT" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + And I send "DELETE" request to "https://default.gw.wso2.com:9095/header-modifier-filters/3.14/employee/1" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And the response body should contain "\"Test-Request-Header\": \"Test-Value\"" + And the response body should contain "\"Set-Request-Header\": \"Test-Value\"" + And the response body should not contain "\"Authorization\"" + Then the response headers contains key "Set-Response-Header" and value "Test-Value" + Then the response headers contains key "Test-Response-Header" and value "Test-Value" + And the response headers should not contain + | content-type | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-header-modifier-filters" + Then the response status code should be 202 + + \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature new file mode 100644 index 0000000000..27e5a37fb6 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/Interceptor.feature @@ -0,0 +1,119 @@ +Feature: API Deployment with Interceptor + Scenario: Deploying an API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/interceptors/original.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + And the response body should not contain "\"Interceptor-Header\"" + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestInterceptor.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptor.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation1.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation2.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation3.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withResponseInterceptorParameterVariation4.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestAndResponse.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + Then I use the APK Conf file "artifacts/apk-confs/interceptors/withRequestAndResponsetls.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + And the response body should contain "\"Interceptor-Header\": \"Interceptor-header-value\"" + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + Scenario Outline: Undeploy an API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | 547961eeaafed989119c45ffc13f8b87bfda821d | 202 | \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/JWT.feature b/test/cucumber-tests/src/test/resources/tests/api/JWT.feature new file mode 100644 index 0000000000..81988e17a2 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/JWT.feature @@ -0,0 +1,126 @@ +Feature: Test JWT related functionalities + Scenario: Test JWT authentication with valid and invalid access token + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/jwt_basic_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + Then I set headers + |Authorization|bearer invalidToken| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And the response status code should be 401 + Then I remove header "Authorization" + Then I set headers + | custom-jwt | ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And the response status code should be 200 + Scenario: Test JWT Token from different issuer with JWKS + Given The system is ready + Then I generate JWT token from idp1 with kid "123-456" + Then I set headers + |Authorization|bearer ${idp-1-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + Then I set headers + |Authorization|bearer "${idp-1-token}h"| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |429| + |200| + Then I set headers + |Authorization|bearer ${idp-1-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + Then I generate JWT token from idp1 with kid "456-789" + And I send "DELETE" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/1234" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + Then I set headers + |Authorization|bearer ${idp-1-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |429| + |200| + Scenario: Test disabled JWT configuration + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/jwt_disabled_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer invalidToken| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-disabled/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + + Scenario: Test customized JWT headers + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/jwt_custom_header_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |429| + |200| + Then I set headers + |testAuth|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And the response body should contain "\"X-Jwt-Assertion\"" + And the "X-Jwt-Assertion" jwt should validate from JWKS "https://api.am.wso2.com:9095/.wellknown/jwks" and contain + | claim | value | + | claim1 | value1 | + | claim2 | value2 | + + Scenario: Test customized JWT headers with Resource Endpoint + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/jwt_custom_header_resource_endpoint_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header-resource/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |429| + |200| + Then I set headers + |testAuth|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-custom-header-resource/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | jwt-basic-test | 202 | + | jwt-disabled-test | 202 | + | jwt-custom-header-test | 202 | + | jwt-custom-header-resource-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature new file mode 100644 index 0000000000..6cd8fb4fe5 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Mandatory.feature @@ -0,0 +1,358 @@ +Feature: Test mTLS between client and gateway with client certificate sent in header + # mTLS mandatory OAuth2 mandatory + Scenario: Test mandatory mTLS and mandatory OAuth2 with a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and mandatory OAuth2 with an invalid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "invalid-cert.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and mandatory OAuth2 without a client certificate in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" + Then the response status code should be 202 + + # mTLS optional OAuth2 mandatory + Scenario: Test optional mTLS and mandatory OAuth2 with a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" + Then the response status code should be 202 + + Scenario: Test optional mTLS and mandatory OAuth2 without a token + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" + Then the response status code should be 202 + + Scenario: Test optional mTLS and mandatory OAuth2 with an invalid token in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-enabled" + Then the response status code should be 202 + + # mTLS optional OAuth2 disabled + Scenario: Test optional mTLS and disabled OAuth2 with a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_disabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-disabled" + Then the response status code should be 202 + + # mTLS disabled OAuth2 optional + Scenario: Test an API with mTLS disabled and OAuth2 optional + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-disabled-oauth2-optional" + Then the response status code should be 202 + + # mTLS disabled OAuth2 disabled + Scenario: Test an API with mTLS disabled and OAuth2 disabled + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_disabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-disabled-oauth2-disabled" + Then the response status code should be 202 + + # mTLS mandatory OAuth2 disabled + Scenario: Test mandatory mTLS and disabled OAuth2 with a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and disabled OAuth2 with an invalid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "invalid-cert.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and disabled OAuth2 without a client certificate in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_disabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-disabled" + Then the response status code should be 202 + + # mTLS disabled OAuth2 mandatory + Scenario: Test an API with mTLS disabled and OAuth2 mandatory + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_disabled_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-disabled-oauth2-enabled" + Then the response status code should be 202 + + # Multiple certificates test cases + Scenario: Test an API with mTLS enabled and one associated certificate with multiple certificates existing in system + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_enabled.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And I have a valid token with a client certificate "config-map-2.txt" + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + And I have a valid token with a client certificate "config-map-3.txt" + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-enabled" + Then the response status code should be 202 + + Scenario: Test an API with mTLS enabled and multiple certificates configured + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_multiple_certs.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And I have a valid token with a client certificate "config-map-2.txt" + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + And I have a valid token with a client certificate "config-map-3.txt" + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-multiple-certs " + Then the response status code should be 202 diff --git a/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature new file mode 100644 index 0000000000..e17fd60308 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/MTLSwithOAuth2Optional.feature @@ -0,0 +1,180 @@ +Feature: Test mTLS between client and gateway with client certificate sent in header with OAuth2 optional + # mTLS mandatory OAuth2 optional + Scenario: Test mandatory mTLS and optional OAuth2 with a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and optional OAuth2 with an invalid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "invalid-cert.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test mandatory mTLS and optional OAuth2 without a client certificate in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_mandatory_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-mandatory-oauth2-optional" + Then the response status code should be 202 + + # mTLS optional OAuth2 optional + Scenario: Test optional mTLS and optional OAuth2 with a valid token and then a valid client certificate in header + Given The system is ready + And I have a valid token with a client certificate "config-map-1.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test optional mTLS and optional OAuth2 with an invalid client certificate and invalid token in header + Given The system is ready + And I have a valid token with a client certificate "invalid-cert.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + | Authorization | bearer {accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test optional mTLS and optional OAuth2 with an invalid client certificate and valid token in header + Given The system is ready + And I have a valid token with a client certificate "invalid-cert.txt" + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + Then I set headers + | X-WSO2-CLIENT-CERTIFICATE | ${clientCertificate} | + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test optional mTLS and optional OAuth2 with an invalid token in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer invalidToken | + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-optional" + Then the response status code should be 202 + + Scenario: Test optional mTLS and optional OAuth2 with no client certificate or token in header + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/mtls/mtls_optional_oauth2_optional.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/mtls/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + | 200 | + + Scenario: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "mtls-optional-oauth2-optional" + Then the response status code should be 202 \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature b/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature new file mode 100644 index 0000000000..7c49dd524b --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/MultiEnvironment.feature @@ -0,0 +1,117 @@ +Feature: Deploy APIs in multiple environments + Scenario: Deploying an API without specifing an Environment and token issuer has no environments. + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I undeploy the API whose ID is "without-env-api" + Then the response status code should be 202 + + Scenario: Deploying an API without specifing an Environment and token issuer has all(*) environments. + Given The system is ready + And I have a valid token for organization "org3" + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org3" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org3}| + And I send "GET" request to "https://org3.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I undeploy the API whose ID is "without-env-api" and organization "org3" + Then the response status code should be 202 + + Scenario: Deploying an API without specifing an Environment and token issuer has only dev environment. + Given The system is ready + And I have a valid token for organization "org4" + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org4" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org4}| + And I send "GET" request to "https://org4.gw.wso2.com:9095/withoutenv/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |200| + When I undeploy the API whose ID is "without-env-api" and organization "org4" + Then the response status code should be 202 + + Scenario: Deploying APIs in Dev and QA environments and token issuer has no environments. + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_dev.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default-dev.gw.wso2.com:9095/multienv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I undeploy the API whose ID is "multi-env-dev-api" + Then the response status code should be 202 + When I undeploy the API whose ID is "multi-env-qa-api" + Then the response status code should be 202 + + Scenario: Deploying an API in QA environment and token issuer has all(*) environments. + Given The system is ready + And I have a valid token for organization "org3" + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org3" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org3}| + And I send "GET" request to "https://org3-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |401| + When I undeploy the API whose ID is "multi-env-qa-api" and organization "org3" + Then the response status code should be 202 + + Scenario: Deploying an API in QA environment and token issuer has only Dev environment. + Given The system is ready + And I have a valid token for organization "org4" + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_qa.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org4" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org4}| + And I send "GET" request to "https://org4-qa.gw.wso2.com:9095/multienv/3.14/employee/" with body "" + And I eventually receive 401 response code, not accepting + |200| + When I undeploy the API whose ID is "multi-env-qa-api" and organization "org4" + Then the response status code should be 202 + + Scenario: Deploying an API in Dev environment and token issuer has only Dev environment. + Given The system is ready + And I have a valid token for organization "org4" + When I use the APK Conf file "artifacts/apk-confs/multi-env/employees_conf_dev.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org4" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org4}| + And I send "GET" request to "https://org4-dev.gw.wso2.com:9095/multienv/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |401| + When I undeploy the API whose ID is "multi-env-dev-api" and organization "org4" + Then the response status code should be 202 + \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature b/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature new file mode 100644 index 0000000000..973ec45068 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/OrganizationBasedAPIS.feature @@ -0,0 +1,47 @@ +Feature: Organization Base API Deployment + Scenario: Deploying an API and basic http method invocations + Given The system is ready + And I have a valid token for organization "org1" + When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org1" + Then the response status code should be 200 + And I have a valid token for organization "org2" + When I use the APK Conf file "artifacts/apk-confs/employees_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request for organization "org2" + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${org1}| + And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + Then I set headers + |Authorization|bearer ${org2}| + And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I undeploy the API whose ID is "432bf873bf751ad714ddc635b0a4d9d194b39eb3" and organization "org1" + Then the response status code should be 202 + Then I set headers + |Authorization|bearer ${org1}| + And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 404 response code, not accepting + |429| + Then I set headers + |Authorization|bearer ${org2}| + And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + When I undeploy the API whose ID is "d45e79bec8c4b2d3a7543e09530e0a995ea68691" and organization "org2" + Then the response status code should be 202 + Then I set headers + |Authorization|bearer ${org1}| + And I send "GET" request to "https://org1.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 404 response code, not accepting + |429| + Then I set headers + |Authorization|bearer ${org2}| + And I send "GET" request to "https://org2.gw.wso2.com:9095/test/3.14/employee/" with body "" + And I eventually receive 404 response code, not accepting + |429| \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature new file mode 100644 index 0000000000..3dfe4f6af3 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestMirror.feature @@ -0,0 +1,21 @@ +Feature: Test HTTPRoute Filter Request Mirror functionality + Scenario: Test request mirror functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-mirror-filter.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-mirror-filter/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-mirror-filter" + Then the response status code should be 202 + + \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature new file mode 100644 index 0000000000..28570f428d --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/RequestRedirect.feature @@ -0,0 +1,51 @@ +Feature: Test HTTPRoute Filter Request Redirect functionality + Scenario: Test request redirect functionality + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/request-redirect-filter.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-redirect-filter" + Then the response status code should be 202 + + Scenario: Test request redirect functionality with API level redirect + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/httproute-filters/api-level-redirect.apk-conf" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + | Authorization | bearer ${accessToken} | + And I send "GET" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "POST" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "PUT" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + And I send "DELETE" request to "https://default.gw.wso2.com:9095/request-redirect-filter/3.14/employee/1" with body "" + And I eventually receive 301 response code, not accepting + | 401 | + + Scenario: Undeploy the API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "api-with-request-redirect-filter" + Then the response status code should be 202 + \ No newline at end of file diff --git a/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature b/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature new file mode 100644 index 0000000000..b4c1adc3a4 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/RevokedToken.feature @@ -0,0 +1,35 @@ +Feature: Token revocation + Scenario: Testing token revocation + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/jwt_basic_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And the response status code should be 200 + # Revoke the token + Then I set headers + |stsAuthKey|2jrmypak391zsqz974ugdddebf812ofx1b9t1oq27530ir02tc815eemrx435qvcp41ucgy7v5uuawzi4qcmjrx0k1zgox2s28cr| + And I send "POST" request to "https://api.am.wso2.com:9095/api/notification/1.0.0/notify?type=TOKEN_REVOCATION" with body "{\"token\": \"${accessToken}\"}" + And the response status code should be 200 + And I wait for 5 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/jwt-basic/3.14/employee/" with body "" + And the response status code should be 401 + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | jwt-basic-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature b/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature new file mode 100644 index 0000000000..6aa4ca1d2a --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/SemanticVersioning.feature @@ -0,0 +1,118 @@ +Feature: Semantic Versioning Based Intelligent Routing + + Scenario: API version with Major and Minor + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-0.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| + + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.0/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.0\"" + + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.0\"" + + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-1.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.1\"" + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.1\"" + + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-5.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1.5/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.5\"" + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.5\"" + + When I undeploy the API whose ID is "sem-api-v1-5" + Then the response status code should be 202 + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + And the response body should contain "\"version\":\"v1.1\"" + + When I undeploy the API whose ID is "sem-api-v1-1" + Then the response status code should be 202 + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + And the response body should contain "\"version\":\"v1.0\"" + + When I undeploy the API whose ID is "sem-api-v1-0" + Then the response status code should be 202 + + Scenario: Multiple Major and minor versions for an API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-0.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-1.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v1-5.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" + Then I set headers + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.5\"" + + When I undeploy the API whose ID is "sem-api-v1-1" + Then the response status code should be 202 + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.5\"" + + When I use the APK Conf file "artifacts/apk-confs/semantic-versioning/sem_api_v2-1.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v2.1\"" + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2.1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v2.1\"" + + When I undeploy the API whose ID is "sem-api-v1-0" + Then the response status code should be 202 + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v1.5\"" + + When I undeploy the API whose ID is "sem-api-v1-5" + Then the response status code should be 202 + And I wait for 2 seconds + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v1/employee/" with body "" + Then the response status code should be 404 + + And I send "GET" request to "https://default.gw.wso2.com:9095/sem-api/v2/employee/" with body "" + Then the response status code should be 200 + And the response body should contain "\"version\":\"v2.1\"" + + When I undeploy the API whose ID is "sem-api-v2-1" + Then the response status code should be 202 diff --git a/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature b/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature new file mode 100644 index 0000000000..7b88660c50 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/SimpleRateLimit.feature @@ -0,0 +1,76 @@ +Feature: Test simple rate limit feature + Scenario: Test simple rate limit api level + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/simple_rl_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I wait for next minute + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" + Then the response status code should be 429 + And I send "GET" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl/3.14/employee/" with body "" + Then the response status code should be 429 + + Scenario: Test simple rate limit api level for unsecured api + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/simple_rl_jwt_disabled_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|invalid| + And I wait for next minute + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-jwt-disabled/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-jwt-disabled/3.14/employee/" with body "" + Then the response status code should be 429 + + Scenario: Test simple rate limit resource level + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/simple_rl_resource_conf.yaml" + And the definition file "artifacts/definitions/employees_api.json" + And make the API deployment request + Then the response status code should be 200 + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "POST" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "POST" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + Then the response status code should be 429 + And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + And I eventually receive 200 response code, not accepting + |429| + |401| + And I send "POST" request to "https://default.sandbox.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + Then the response status code should be 429 + And I wait for next minute + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + Then the response status code should be 200 + And I send "GET" request to "https://default.gw.wso2.com:9095/simple-rl-r/3.14/employee/" with body "" + Then the response status code should be 200 + + + Scenario Outline: Undeploy API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | simple-rl-test | 202 | + | simple-rl-r-test | 202 | + | simple-rl-jwt-disabled-test | 202 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/deployment.feature b/test/cucumber-tests/src/test/resources/tests/api/deployment.feature new file mode 100644 index 0000000000..56e2ef2605 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/deployment.feature @@ -0,0 +1,44 @@ +Feature: API Deployment + Scenario: Deploying an API without api create scope + Given The system is ready + And I have a valid subscription without api deploy permission + When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 403 + + Scenario: Deploying an API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/cors_API.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "cors-api-adff3dbc-2787-11ee-be56-0242ac120002" + + Scenario: Deploying an API with invalid APK Conf file + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/invalid_cors_API.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 400 + And the response body should contain + |"#/corsConfiguration/corsConfigurationEnabled: expected type: Boolean, found: String"| + + Scenario Outline: Undeploy an API without api create scope + Given The system is ready + And I have a valid subscription without api deploy permission + When I undeploy the API whose ID is "" + Then the response status code should be 403 + + Scenario Outline: Undeploy an API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | cors-api-adff3dbc-2787-11ee-be56-0242ac120002 | 202 | + | abcdeadsxzads | 404 | diff --git a/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature b/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature new file mode 100644 index 0000000000..93271f5287 --- /dev/null +++ b/test/cucumber-tests/src/test/resources/tests/api/resourceInterceptor.feature @@ -0,0 +1,34 @@ +Feature: API Deployment with Resource Interceptor + Scenario: Deploying an API + Given The system is ready + And I have a valid subscription + When I use the APK Conf file "artifacts/apk-confs/interceptors/resourceLevelInterptor.apk-conf" + And the definition file "artifacts/definitions/cors_api.yaml" + And make the API deployment request + Then the response status code should be 200 + And the response body should contain "547961eeaafed989119c45ffc13f8b87bfda821d" + And I wait for 1 minute + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/get" with body "" + And the response body should not contain "\"Interceptor-Header\"" + Then the response status code should be 200 + Then the response headers not contains key "interceptor-response-header" + Then I set headers + |Authorization|bearer ${accessToken}| + And I send "GET" request to "https://default.gw.wso2.com:9095/interceptor/1.0.0/headers" with body "" + And the response body should contain + |"Interceptor-Header": "Interceptor-header-value"| + |"Interceptor-Header-Apigroup": "Gold"| + |"Interceptor-Header-Apitier": "Unlimited"| + Then the response status code should be 200 + Then the response headers contains key "interceptor-response-header" and value "Interceptor-Response-header-value" + Scenario Outline: Undeploy an API + Given The system is ready + And I have a valid subscription + When I undeploy the API whose ID is "" + Then the response status code should be + + Examples: + | apiID | expectedStatusCode | + | 547961eeaafed989119c45ffc13f8b87bfda821d | 202 | \ No newline at end of file From 128d1d816921c9c8403446e6130dcf60e075f927 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 15:34:24 +0530 Subject: [PATCH 19/24] Fix test --- test/cucumber-tests/CRs/llm-test-header.yaml | 163 --------------- test/cucumber-tests/CRs/llm-test-subs.yaml | 200 ------------------- test/cucumber-tests/CRs/llm-test.yaml | 163 --------------- test/cucumber-tests/scripts/setup-hosts.sh | 4 + 4 files changed, 4 insertions(+), 526 deletions(-) delete mode 100644 test/cucumber-tests/CRs/llm-test-header.yaml delete mode 100644 test/cucumber-tests/CRs/llm-test-subs.yaml delete mode 100644 test/cucumber-tests/CRs/llm-test.yaml diff --git a/test/cucumber-tests/CRs/llm-test-header.yaml b/test/cucumber-tests/CRs/llm-test-header.yaml deleted file mode 100644 index 7fd2bff518..0000000000 --- a/test/cucumber-tests/CRs/llm-test-header.yaml +++ /dev/null @@ -1,163 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: llm-deployment-header - namespace: apk-integration-test -spec: - progressDeadlineSeconds: 600 - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - app: llm-app-header - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - app: llm-app-header - spec: - containers: - - image: tharsanan/llm-backend:latest - imagePullPolicy: Always - name: llm-backend-container-header - ports: - - containerPort: 80 - protocol: TCP - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: llm-service-header - namespace: apk-integration-test -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: llm-app-header - type: ClusterIP - ---- - -apiVersion: dp.wso2.com/v1alpha2 -kind: API -metadata: - name: llm-api-header - namespace: apk-integration-test -spec: - apiName: llm-api-header - apiType: REST - apiVersion: v1.0.0 - basePath: /llm-api-header/v1.0.0 - isDefaultVersion: true - production: - - routeRefs: - - llm-route-header - organization: default ---- -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: HTTPRoute -metadata: - name: llm-route-header - namespace: apk-integration-test -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: wso2-apk-default - namespace: apk-integration-test - sectionName: httpslistener - hostnames: - - default.gw.wso2.com - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: dp.wso2.com - kind: Backend - name: llm-backend-header ---- -apiVersion: dp.wso2.com/v1alpha2 -kind: Backend -metadata: - name: llm-backend-header - namespace: apk-integration-test -spec: - services: - - host: llm-service-header - port: 80 ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIRateLimitPolicy -metadata: - name: llm-backend-rl-header - namespace: apk-integration-test -spec: - override: - organization: default - tokenCount: - unit: Minute - requestTokenCount: 5000 - responseTokenCount: 10000 - totalTokenCount: 15000 - requestCount: - requestsPerUnit: 6000 - unit: Minute - targetRef: - kind: Backend - name: llm-backend-header - group: gateway.networking.k8s.io ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: APIPolicy -metadata: - name: llm-policy-header - namespace: apk-integration-test -spec: - override: - aiProvider: - name: "llm-provider-header" - targetRef: - group: gateway.networking.k8s.io - kind: API - name: llm-api-header ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIProvider -metadata: - name: llm-provider-header - namespace: apk-integration-test -spec: - providerName : "AzureAI" - providerAPIVersion : "2024-06-01" - organization : "default" - model: - in: "Header" - value: "model" - rateLimitFields: - promptTokens: - in: "Header" - value: "prompt_tokens" - completionToken: - in: "Header" - value: "completion_tokens" - totalToken: - in: "Header" - value: "total_tokens" diff --git a/test/cucumber-tests/CRs/llm-test-subs.yaml b/test/cucumber-tests/CRs/llm-test-subs.yaml deleted file mode 100644 index b33c66ca27..0000000000 --- a/test/cucumber-tests/CRs/llm-test-subs.yaml +++ /dev/null @@ -1,200 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: llm-deployment-subs - namespace: apk-integration-test -spec: - progressDeadlineSeconds: 600 - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - app: llm-app-subs - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - app: llm-app-subs - spec: - containers: - - image: tharsanan/llm-backend:latest - imagePullPolicy: Always - name: llm-backend-container-subs - ports: - - containerPort: 80 - protocol: TCP - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: llm-service-subs - namespace: apk-integration-test -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: llm-app-subs - type: ClusterIP - ---- - -apiVersion: dp.wso2.com/v1alpha2 -kind: API -metadata: - name: llm-api-subs - namespace: apk-integration-test -spec: - apiName: llm-api-subs - apiType: REST - apiVersion: v1.0.0 - basePath: /llm-api-subs/v1.0.0 - isDefaultVersion: true - production: - - routeRefs: - - llm-route-subs - organization: default ---- -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: HTTPRoute -metadata: - name: llm-route-subs - namespace: apk-integration-test -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: wso2-apk-default - namespace: apk-integration-test - sectionName: httpslistener - hostnames: - - default.gw.wso2.com - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: dp.wso2.com - kind: Backend - name: llm-backend-subs ---- -apiVersion: dp.wso2.com/v1alpha2 -kind: Backend -metadata: - name: llm-backend-subs - namespace: apk-integration-test -spec: - services: - - host: llm-service-subs - port: 80 ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIRateLimitPolicy -metadata: - name: llm-backend-rl-subs - namespace: apk-integration-test -spec: - override: - organization: default - tokenCount: - unit: Minute - requestTokenCount: 5000 - responseTokenCount: 10000 - totalTokenCount: 15000 - requestCount: - requestsPerUnit: 6000 - unit: Minute ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: APIPolicy -metadata: - name: llm-policy-subs - namespace: apk-integration-test -spec: - override: - aiProvider: - name: "llm-provider-subs" - subscriptionValidation: true - targetRef: - group: gateway.networking.k8s.io - kind: API - name: llm-api-subs ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIProvider -metadata: - name: llm-provider-subs - namespace: apk-integration-test -spec: - providerName : "AzureAI" - providerAPIVersion : "2024-06-01" - organization : "default" - model: - in: "Body" - value: "model" - rateLimitFields: - promptTokens: - in: "Body" - value: "usage.prompt_tokens" - completionToken: - in: "Body" - value: "usage.completion_tokens" - totalToken: - in: "Body" - value: "usage.total_tokens" ---- -apiVersion: cp.wso2.com/v1alpha2 -kind: Subscription -metadata: - name: llm-subs - namespace: apk-integration-test -spec: - organization: "default" - subscriptionStatus: "ACTIVE" - api: - name: "llm-api-subs" - version: "v1.0.0" - ratelimitRef: - name : llm-backend-rl-subs - level: application ---- -apiVersion: cp.wso2.com/v1alpha2 -kind: ApplicationMapping -metadata: - name: llm-app-map - namespace: apk-integration-test -spec: - applicationRef: llm-app - subscriptionRef: llm-subs ---- -kind: Application -apiVersion: cp.wso2.com/v1alpha2 -metadata: - name: llm-app - namespace : apk-integration-test -spec: - name: sample-app - owner: admin - organization: default - securitySchemes: - oauth2: - environments: - - envId: Default - appId: 45f1c5c8-a92e-11ed-afa1-0242ac120005 - keyType: PRODUCTION \ No newline at end of file diff --git a/test/cucumber-tests/CRs/llm-test.yaml b/test/cucumber-tests/CRs/llm-test.yaml deleted file mode 100644 index d45cda6815..0000000000 --- a/test/cucumber-tests/CRs/llm-test.yaml +++ /dev/null @@ -1,163 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: llm-deployment - namespace: apk-integration-test -spec: - progressDeadlineSeconds: 600 - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - app: llm-app - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate - template: - metadata: - creationTimestamp: null - labels: - app: llm-app - spec: - containers: - - image: tharsanan/llm-backend:latest - imagePullPolicy: Always - name: llm-backend-container - ports: - - containerPort: 80 - protocol: TCP - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: llm-service - namespace: apk-integration-test -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 80 - selector: - app: llm-app - type: ClusterIP - ---- - -apiVersion: dp.wso2.com/v1alpha2 -kind: API -metadata: - name: llm-api - namespace: apk-integration-test -spec: - apiName: llm-api - apiType: REST - apiVersion: v1.0.0 - basePath: /llm-api/v1.0.0 - isDefaultVersion: true - production: - - routeRefs: - - llm-route - organization: default ---- -apiVersion: gateway.networking.k8s.io/v1beta1 -kind: HTTPRoute -metadata: - name: llm-route - namespace: apk-integration-test -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: wso2-apk-default - namespace: apk-integration-test - sectionName: httpslistener - hostnames: - - default.gw.wso2.com - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - group: dp.wso2.com - kind: Backend - name: llm-backend ---- -apiVersion: dp.wso2.com/v1alpha2 -kind: Backend -metadata: - name: llm-backend - namespace: apk-integration-test -spec: - services: - - host: llm-service - port: 80 ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIRateLimitPolicy -metadata: - name: llm-backend-rl - namespace: apk-integration-test -spec: - override: - organization: default - tokenCount: - unit: Minute - requestTokenCount: 5000 - responseTokenCount: 10000 - totalTokenCount: 15000 - requestCount: - requestsPerUnit: 6000 - unit: Minute - targetRef: - kind: Backend - name: llm-backend - group: gateway.networking.k8s.io ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: APIPolicy -metadata: - name: llm-policy - namespace: apk-integration-test -spec: - override: - aiProvider: - name: "llm-provider" - targetRef: - group: gateway.networking.k8s.io - kind: API - name: llm-api ---- -apiVersion: dp.wso2.com/v1alpha3 -kind: AIProvider -metadata: - name: llm-provider - namespace: apk-integration-test -spec: - providerName : "AzureAI" - providerAPIVersion : "2024-06-01" - organization : "default" - model: - in: "Body" - value: "model" - rateLimitFields: - promptTokens: - in: "Body" - value: "usage.prompt_tokens" - completionToken: - in: "Body" - value: "usage.completion_tokens" - totalToken: - in: "Body" - value: "usage.total_tokens" \ No newline at end of file diff --git a/test/cucumber-tests/scripts/setup-hosts.sh b/test/cucumber-tests/scripts/setup-hosts.sh index 5fb4512483..ac372205dc 100644 --- a/test/cucumber-tests/scripts/setup-hosts.sh +++ b/test/cucumber-tests/scripts/setup-hosts.sh @@ -1,9 +1,13 @@ #!/usr/bin/env bash kubectl apply -f ./CRs/artifacts.yaml + kubectl wait deployment/httpbin -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/backend-retry-deployment -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/dynamic-backend -n apk-integration-test --for=condition=available --timeout=600s +kubectl wait deployment/llm-deployment -n apk-integration-test --for=condition=available --timeout=600s +kubectl wait deployment/llm-provider-subs -n apk-integration-test --for=condition=available --timeout=600s +kubectl wait deployment/llm-deployment-header -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/interceptor-service-deployment -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/graphql-faker -n apk-integration-test --for=condition=available --timeout=600s kubectl wait --timeout=5m -n apk-integration-test deployment/apk-test-setup-wso2-apk-adapter-deployment --for=condition=Available From 0b783de54eb5e66624b50af4b3d0b2fb64730459 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 15:34:58 +0530 Subject: [PATCH 20/24] Add artifacts --- test/cucumber-tests/CRs/artifacts.yaml | 528 +++++++++++++++++++++++++ 1 file changed, 528 insertions(+) diff --git a/test/cucumber-tests/CRs/artifacts.yaml b/test/cucumber-tests/CRs/artifacts.yaml index 7b5eba8132..884fcbc485 100644 --- a/test/cucumber-tests/CRs/artifacts.yaml +++ b/test/cucumber-tests/CRs/artifacts.yaml @@ -1194,3 +1194,531 @@ type: Opaque data: apiKey: c2FtcGF0aA== --- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment-subs + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app-subs + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app-subs + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container-subs + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service-subs + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app-subs + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api-subs + namespace: apk-integration-test +spec: + apiName: llm-api-subs + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api-subs/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route-subs + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route-subs + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend-subs +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend-subs + namespace: apk-integration-test +spec: + services: + - host: llm-service-subs + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl-subs + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy-subs + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider-subs" + subscriptionValidation: true + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api-subs +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider-subs + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Body" + value: "model" + rateLimitFields: + promptTokens: + in: "Body" + value: "usage.prompt_tokens" + completionToken: + in: "Body" + value: "usage.completion_tokens" + totalToken: + in: "Body" + value: "usage.total_tokens" +--- +apiVersion: cp.wso2.com/v1alpha2 +kind: Subscription +metadata: + name: llm-subs + namespace: apk-integration-test +spec: + organization: "default" + subscriptionStatus: "ACTIVE" + api: + name: "llm-api-subs" + version: "v1.0.0" + ratelimitRef: + name : llm-backend-rl-subs + level: application +--- +apiVersion: cp.wso2.com/v1alpha2 +kind: ApplicationMapping +metadata: + name: llm-app-map + namespace: apk-integration-test +spec: + applicationRef: llm-app + subscriptionRef: llm-subs +--- +kind: Application +apiVersion: cp.wso2.com/v1alpha2 +metadata: + name: llm-app + namespace : apk-integration-test +spec: + name: sample-app + owner: admin + organization: default + securitySchemes: + oauth2: + environments: + - envId: Default + appId: 45f1c5c8-a92e-11ed-afa1-0242ac120005 + keyType: PRODUCTION +--- +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment-header + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app-header + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app-header + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container-header + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service-header + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app-header + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api-header + namespace: apk-integration-test +spec: + apiName: llm-api-header + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api-header/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route-header + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route-header + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend-header +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend-header + namespace: apk-integration-test +spec: + services: + - host: llm-service-header + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl-header + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute + targetRef: + kind: Backend + name: llm-backend-header + group: gateway.networking.k8s.io +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy-header + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider-header" + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api-header +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider-header + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Header" + value: "model" + rateLimitFields: + promptTokens: + in: "Header" + value: "prompt_tokens" + completionToken: + in: "Header" + value: "completion_tokens" + totalToken: + in: "Header" + value: "total_tokens" +--- +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: llm-deployment + namespace: apk-integration-test +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: llm-app + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: llm-app + spec: + containers: + - image: tharsanan/llm-backend:latest + imagePullPolicy: Always + name: llm-backend-container + ports: + - containerPort: 80 + protocol: TCP + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: llm-service + namespace: apk-integration-test +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: llm-app + type: ClusterIP + +--- + +apiVersion: dp.wso2.com/v1alpha2 +kind: API +metadata: + name: llm-api + namespace: apk-integration-test +spec: + apiName: llm-api + apiType: REST + apiVersion: v1.0.0 + basePath: /llm-api/v1.0.0 + isDefaultVersion: true + production: + - routeRefs: + - llm-route + organization: default +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: llm-route + namespace: apk-integration-test +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: wso2-apk-default + namespace: apk-integration-test + sectionName: httpslistener + hostnames: + - default.gw.wso2.com + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: dp.wso2.com + kind: Backend + name: llm-backend +--- +apiVersion: dp.wso2.com/v1alpha2 +kind: Backend +metadata: + name: llm-backend + namespace: apk-integration-test +spec: + services: + - host: llm-service + port: 80 +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + name: llm-backend-rl + namespace: apk-integration-test +spec: + override: + organization: default + tokenCount: + unit: Minute + requestTokenCount: 5000 + responseTokenCount: 10000 + totalTokenCount: 15000 + requestCount: + requestsPerUnit: 6000 + unit: Minute + targetRef: + kind: Backend + name: llm-backend + group: gateway.networking.k8s.io +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: APIPolicy +metadata: + name: llm-policy + namespace: apk-integration-test +spec: + override: + aiProvider: + name: "llm-provider" + targetRef: + group: gateway.networking.k8s.io + kind: API + name: llm-api +--- +apiVersion: dp.wso2.com/v1alpha3 +kind: AIProvider +metadata: + name: llm-provider + namespace: apk-integration-test +spec: + providerName : "AzureAI" + providerAPIVersion : "2024-06-01" + organization : "default" + model: + in: "Body" + value: "model" + rateLimitFields: + promptTokens: + in: "Body" + value: "usage.prompt_tokens" + completionToken: + in: "Body" + value: "usage.completion_tokens" + totalToken: + in: "Body" + value: "usage.total_tokens" +--- From 8708cc0ef5c1a56e1d8f3d54318e1294031c3795 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Wed, 18 Sep 2024 15:43:53 +0530 Subject: [PATCH 21/24] Fix intermittent issue --- adapter/internal/discovery/xds/server.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/adapter/internal/discovery/xds/server.go b/adapter/internal/discovery/xds/server.go index 2c45800c78..2336d99417 100644 --- a/adapter/internal/discovery/xds/server.go +++ b/adapter/internal/discovery/xds/server.go @@ -566,13 +566,9 @@ func PopulateInternalMaps(adapterInternalAPI *model.AdapterInternalAPI, labels, for vhost := range vHosts { if adapterInternalAPI.AIProvider.Enabled && adapterInternalAPI.GetSubscriptionValidation() { vHostToSubscriptionBasedAIRLMap[vhost] = true - } else { - vHostToSubscriptionBasedAIRLMap[vhost] = false } if adapterInternalAPI.GetSubscriptionValidation() { vHostToSubscriptionBasedRLMap[vhost] = true - } else { - vHostToSubscriptionBasedRLMap[vhost] = false } } if err != nil { From 5cd8cb4666ba28623769ff56e967d0fbf7a9baf3 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Thu, 19 Sep 2024 09:14:18 +0530 Subject: [PATCH 22/24] Fix nil pointer err --- adapter/internal/operator/synchronizer/data_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 995aff427f..dd42b2ef75 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -169,7 +169,7 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced break } } - if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { + if len(cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { updated = true } } From 326de22cc16eb2a7e605a15e8b90eef1f498fc6f Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Thu, 19 Sep 2024 11:15:43 +0530 Subject: [PATCH 23/24] Change app id --- test/cucumber-tests/CRs/artifacts.yaml | 11 +++++++++-- test/cucumber-tests/scripts/setup-hosts.sh | 2 +- .../tests/api/APISubscriptionBasedAIRatelimit.feature | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/cucumber-tests/CRs/artifacts.yaml b/test/cucumber-tests/CRs/artifacts.yaml index 884fcbc485..26ebb0248c 100644 --- a/test/cucumber-tests/CRs/artifacts.yaml +++ b/test/cucumber-tests/CRs/artifacts.yaml @@ -1194,6 +1194,13 @@ type: Opaque data: apiKey: c2FtcGF0aA== --- + + + + + + + apiVersion: apps/v1 kind: Deployment metadata: @@ -1384,14 +1391,14 @@ metadata: name: llm-app namespace : apk-integration-test spec: - name: sample-app + name: sample-llm-app owner: admin organization: default securitySchemes: oauth2: environments: - envId: Default - appId: 45f1c5c8-a92e-11ed-afa1-0242ac120005 + appId: 45f1c5c8-a92e-11ed-afa1-0242ac120065 keyType: PRODUCTION --- --- diff --git a/test/cucumber-tests/scripts/setup-hosts.sh b/test/cucumber-tests/scripts/setup-hosts.sh index ac372205dc..5b307165c2 100644 --- a/test/cucumber-tests/scripts/setup-hosts.sh +++ b/test/cucumber-tests/scripts/setup-hosts.sh @@ -6,7 +6,7 @@ kubectl wait deployment/httpbin -n apk-integration-test --for=condition=availabl kubectl wait deployment/backend-retry-deployment -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/dynamic-backend -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/llm-deployment -n apk-integration-test --for=condition=available --timeout=600s -kubectl wait deployment/llm-provider-subs -n apk-integration-test --for=condition=available --timeout=600s +kubectl wait deployment/llm-deployment-subs -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/llm-deployment-header -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/interceptor-service-deployment -n apk-integration-test --for=condition=available --timeout=600s kubectl wait deployment/graphql-faker -n apk-integration-test --for=condition=available --timeout=600s diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature index 7f908508cd..600ae88faf 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature @@ -2,7 +2,7 @@ Feature: API subscription based AI ratelimit Feature Scenario: subscription based AI ratelimit token detail comes in the body. Given The system is ready And I have a valid subscription - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120065" Then I set headers |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| And I wait for next minute strictly @@ -39,7 +39,7 @@ Feature: API subscription based AI ratelimit Feature Scenario: subs based AI ratelimit token detail comes in the header but a body configured api checked. Given The system is ready And I have a valid subscription - Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120005" + Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120065" Then I set headers |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| And I wait for next minute strictly From 182d0358539af5d14aaeadcd581fe8d02c58ccab Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Thu, 19 Sep 2024 11:57:42 +0530 Subject: [PATCH 24/24] Fix test fails --- .../tests/api/APISubscriptionBasedAIRatelimit.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature index 600ae88faf..3209f9360b 100644 --- a/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature +++ b/test/cucumber-tests/src/test/resources/tests/api/APISubscriptionBasedAIRatelimit.feature @@ -4,7 +4,7 @@ Feature: API subscription based AI ratelimit Feature And I have a valid subscription Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120065" Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120065-token}| And I wait for next minute strictly And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=body" with body "" Then the response status code should be 200 @@ -41,7 +41,7 @@ Feature: API subscription based AI ratelimit Feature And I have a valid subscription Then I generate JWT token from idp1 with kid "123-456" and consumer_key "45f1c5c8-a92e-11ed-afa1-0242ac120065" Then I set headers - |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120005-token}| + |Authorization|bearer ${idp-1-45f1c5c8-a92e-11ed-afa1-0242ac120065-token}| And I wait for next minute strictly And I send "GET" request to "https://default.gw.wso2.com:9095/llm-api-subs/v1.0.0/3.14/employee?send=header" with body "" Then the response status code should be 200