From 042d0154fc46dc5a48941523835fa8136be8e175 Mon Sep 17 00:00:00 2001 From: Tharsanan1 Date: Tue, 27 Aug 2024 09:38:03 +0530 Subject: [PATCH] Enforcer initial changes --- adapter/go.mod | 32 +- adapter/go.sum | 82 ++--- .../internal/oasparser/envoyconf/constants.go | 1 + .../oasparser/envoyconf/http_filters.go | 44 +++ .../oasparser/envoyconf/routes_configs.go | 153 ++++++++- .../envoyconf/routes_with_clusters.go | 116 ++++++- .../oasparser/model/adapter_internal_api.go | 29 +- adapter/internal/oasparser/model/resource.go | 36 ++- adapter/internal/operator/PROJECT | 9 + .../operator/config/crd/kustomization.yaml | 1 + ...cainjection_in_dp_airatelimitpolicies.yaml | 7 + .../webhook_in_dp_airatelimitpolicies.yaml | 16 + .../dp_airatelimitpolicy_editor_role.yaml | 31 ++ .../dp_airatelimitpolicy_viewer_role.yaml | 27 ++ .../internal/operator/constants/constants.go | 9 +- .../operator/controllers/dp/api_controller.go | 240 ++++++++++++-- .../operator/controllers/dp/suite_test.go | 4 + adapter/internal/operator/operator.go | 4 + .../operator/synchronizer/api_state.go | 2 + .../operator/synchronizer/data_store.go | 42 +++ .../operator/synchronizer/rest_api.go | 8 +- adapter/internal/operator/utils/utils.go | 5 + common-controller/go.mod | 36 +-- common-controller/go.sum | 90 +++--- common-controller/internal/cache/datastore.go | 26 ++ .../operator/config/crd/kustomization.yaml | 1 + .../dp/airatelimitpolicy_controller.go | 159 +++++++++ .../operator/controllers/dp/suite_test.go | 8 + .../internal/operator/operator.go | 5 + .../internal/xds/ratelimiter_cache.go | 84 ++++- common-controller/internal/xds/server.go | 11 + common-go-libs/PROJECT | 13 + .../dp/v1alpha3/airatelimitpolicy_types.go | 98 ++++++ .../apis/dp/v1alpha3/zz_generated.deepcopy.go | 140 ++++++++ .../dp.wso2.com_airatelimitpolicies.yaml | 185 +++++++++++ common-go-libs/config/crd/kustomization.yaml | 3 + ...cainjection_in_dp_airatelimitpolicies.yaml | 7 + .../webhook_in_dp_airatelimitpolicies.yaml | 16 + .../dp_airatelimitpolicy_editor_role.yaml | 31 ++ .../dp_airatelimitpolicy_viewer_role.yaml | 27 ++ common-go-libs/config/rbac/role.yaml | 32 ++ .../dp_v1alpha3_airatelimitpolicy.yaml | 12 + common-go-libs/constants/constant.go | 1 + .../apk/enforcer/config/ConfigHolder.java | 2 + .../apk/enforcer/config/EnvVarConfig.java | 21 ++ .../grpc/ExternalProcessorService.java | 303 ++++++++++++++++++ .../enforcer/grpc/client/RatelimitClient.java | 97 ++++++ .../wso2/apk/enforcer/server/AuthServer.java | 3 + gateway/router/Dockerfile | 2 +- .../crds/dp.wso2.com_airatelimitpolicies.yaml | 201 ++++++++++++ .../gateway-runtime-deployment.yaml | 24 +- .../serviceAccount/apk-cluster-role.yaml | 9 + libs.versions.toml | 4 +- 53 files changed, 2344 insertions(+), 205 deletions(-) create mode 100644 adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml create mode 100644 adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml create mode 100644 adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml create mode 100644 adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml create mode 100644 common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go create mode 100644 common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go create mode 100644 common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml create mode 100644 common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml create mode 100644 common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml create mode 100644 common-go-libs/config/rbac/role.yaml create mode 100644 common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml create mode 100644 gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java create mode 100644 gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java create mode 100644 helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml diff --git a/adapter/go.mod b/adapter/go.mod index eeb7f4350b..bf720480fd 100644 --- a/adapter/go.mod +++ b/adapter/go.mod @@ -3,9 +3,9 @@ module github.com/wso2/apk/adapter go 1.22 require ( - github.com/envoyproxy/go-control-plane v0.12.0 + github.com/envoyproxy/go-control-plane v0.13.0 github.com/fsnotify/fsnotify v1.7.0 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 @@ -14,7 +14,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/wso2/apk/common-go-libs v0.0.0-20231208100153-24bee7b4bd81 golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb - google.golang.org/grpc v1.62.0 + google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.29.2 @@ -24,11 +24,12 @@ require ( ) require ( + cel.dev/expr v0.15.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -56,9 +57,10 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect @@ -70,17 +72,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.2 // indirect @@ -99,8 +99,8 @@ replace github.com/wso2/apk/common-go-libs => ../common-go-libs require ( github.com/ghodss/yaml v1.0.0 - github.com/stretchr/testify v1.8.4 - golang.org/x/sys v0.17.0 // indirect + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.20.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 sigs.k8s.io/gateway-api v1.0.0 ) diff --git a/adapter/go.sum b/adapter/go.sum index 9bc0af125c..5eaa7328cd 100644 --- a/adapter/go.sum +++ b/adapter/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= @@ -9,13 +11,13 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -25,8 +27,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= -github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= @@ -57,13 +59,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -112,6 +111,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -119,8 +120,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -149,8 +150,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -159,7 +161,6 @@ github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqf github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -171,51 +172,39 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,7 +212,6 @@ golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -232,18 +220,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/adapter/internal/oasparser/envoyconf/constants.go b/adapter/internal/oasparser/envoyconf/constants.go index 1798d6d19a..c92501c054 100644 --- a/adapter/internal/oasparser/envoyconf/constants.go +++ b/adapter/internal/oasparser/envoyconf/constants.go @@ -29,6 +29,7 @@ const ( const ( httpConManagerStartPrefix string = "ingress_http" extAuthzPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute" + extProcPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute" luaPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute" corsFilterName string = "type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors" localRateLimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit" diff --git a/adapter/internal/oasparser/envoyconf/http_filters.go b/adapter/internal/oasparser/envoyconf/http_filters.go index 4faad45e96..06867d6c76 100644 --- a/adapter/internal/oasparser/envoyconf/http_filters.go +++ b/adapter/internal/oasparser/envoyconf/http_filters.go @@ -28,6 +28,7 @@ import ( envoy_config_ratelimit_v3 "github.com/envoyproxy/go-control-plane/envoy/config/ratelimit/v3" cors_filter_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" ext_authv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + ext_process "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" luav3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" ratelimit "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ratelimit/v3" routerv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" @@ -47,9 +48,14 @@ import ( "github.com/golang/protobuf/ptypes/any" ) + +// HTTPExternalProcessor HTTP filter +const HTTPExternalProcessor = "envoy.filters.http.ext_proc" + // getHTTPFilters generates httpFilter configuration func getHTTPFilters(globalLuaScript string) []*hcmv3.HttpFilter { extAuth := getExtAuthzHTTPFilter() + extProcessor := getExtProcessHTTPFilter() router := getRouterHTTPFilter() luaLocal := getLuaFilter(LuaLocal, ` function envoy_on_request(request_handle) @@ -64,6 +70,7 @@ end`) extAuth, luaLocal, luaGlobal, + extProcessor, } conf := config.ReadConfigs() if conf.Envoy.RateLimit.Enabled { @@ -190,6 +197,43 @@ func getRateLimitFilter() *hcmv3.HttpFilter { return &rlFilter } +// getExtProcessHTTPFilter gets ExtAauthz http filter. +func getExtProcessHTTPFilter() *hcmv3.HttpFilter { + // conf := config.ReadConfigs() + externalProcessor := &ext_process.ExternalProcessor{ + GrpcService: &corev3.GrpcService{ + TargetSpecifier: &corev3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &corev3.GrpcService_EnvoyGrpc{ + ClusterName: extAuthzClusterName, + }, + }, + }, + ProcessingMode: &ext_process.ProcessingMode{ + ResponseBodyMode: ext_process.ProcessingMode_BUFFERED, + RequestHeaderMode: ext_process.ProcessingMode_SKIP, + ResponseHeaderMode: ext_process.ProcessingMode_SKIP, + }, + MetadataOptions: &ext_process.MetadataOptions{ + ForwardingNamespaces: &ext_process.MetadataOptions_MetadataNamespaces{ + Untyped: []string{"envoy.filters.http.ext_authz", "envoy.filters.http.ext_proc"}, + }, + }, + RequestAttributes: []string{"xds.route_metadata"}, + ResponseAttributes: []string{"xds.route_metadata"}, + } + ext, err2 := anypb.New(externalProcessor) + if err2 != nil { + logger.LoggerOasparser.Error(err2) + } + extProcessFilter := hcmv3.HttpFilter{ + Name: HTTPExternalProcessor, + ConfigType: &hcmv3.HttpFilter_TypedConfig{ + TypedConfig: ext, + }, + } + return &extProcessFilter +} + // getExtAuthzHTTPFilter gets ExtAauthz http filter. func getExtAuthzHTTPFilter() *hcmv3.HttpFilter { conf := config.ReadConfigs() diff --git a/adapter/internal/oasparser/envoyconf/routes_configs.go b/adapter/internal/oasparser/envoyconf/routes_configs.go index 12b87c0a25..797193e7ad 100644 --- a/adapter/internal/oasparser/envoyconf/routes_configs.go +++ b/adapter/internal/oasparser/envoyconf/routes_configs.go @@ -27,6 +27,7 @@ import ( corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" envoy_type_matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" metadatav3 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" "github.com/envoyproxy/go-control-plane/pkg/wellknown" @@ -40,6 +41,11 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + v35 "github.com/envoyproxy/go-control-plane/envoy/type/metadata/v3" +) + +const( + authzNamespace = "envoy.filters.http.ext_authz" ) // Constants for Rate Limiting @@ -97,7 +103,7 @@ func generateRouteMatch(routeRegex string) *routev3.RouteMatch { return match } -func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string) (action *routev3.Route_Route) { +func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, ratelimitCriteria *ratelimitCriteria, mirrorClusterNames []string, isSubscriptionBasedAIRatelimitEnabled bool, isBackendBasedAIRatelimitEnabled bool, descriptorValueForBackendBasedAIRatelimit string) (action *routev3.Route_Route) { action = &routev3.Route_Route{ Route: &routev3.RouteAction{ HostRewriteSpecifier: &routev3.RouteAction_AutoHostRewrite{ @@ -128,6 +134,9 @@ func generateRouteAction(apiType string, routeConfig *model.EndpointConfig, rate if ratelimitCriteria != nil && ratelimitCriteria.level != "" { action.Route.RateLimits = generateRateLimitPolicy(ratelimitCriteria) } + if isBackendBasedAIRatelimitEnabled { + action.Route.RateLimits = append(action.Route.RateLimits, generateBackendBasedAIRatelimit(descriptorValueForBackendBasedAIRatelimit)...) + } // Add request mirroring configurations if mirrorClusterNames != nil && len(mirrorClusterNames) > 0 { @@ -182,6 +191,136 @@ func mapStatusCodeToEnum(statusCode int) int { return -1 } } +const ( + // DescriptorKeyForAIRequestTokenCount is the descriptor key for AI request token count ratelimit + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + // DescriptorKeyForAIResponseTokenCount is the descriptor key for AI response token count ratelimit + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + // DescriptorKeyForAITotalTokenCount is the descriptor key for AI total token count ratelimit + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + // DescriptorKeyForAIRequestCount is the descriptor key for AI request count ratelimit + DescriptorKeyForAIRequestCount = "airequestcount" +) + +func generateBackendBasedAIRatelimit(descValue string) []*routev3.RateLimit { + rateLimitForRequestTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIRequestTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForResponseTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIResponseTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForTotalTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAITotalTokenCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + rateLimitForRequestCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_GenericKey_{ + GenericKey: &routev3.RateLimit_Action_GenericKey{ + DescriptorKey: DescriptorKeyForAIRequestCount, + DescriptorValue: descValue, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount, &rateLimitForTotalTokenCount} +} + + +func generateSubscriptionBasedAIRatelimit(descValue string) []*routev3.RateLimit { + rateLimitForRequestTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestTokenCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + rateLimitForResponseTokenCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIResponseTokenCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + rateLimitForRequestCount := routev3.RateLimit{ + Actions: []*routev3.RateLimit_Action{ + { + ActionSpecifier: &routev3.RateLimit_Action_Metadata{ + Metadata: &routev3.RateLimit_Action_MetaData{ + DescriptorKey: DescriptorKeyForAIRequestCount, + MetadataKey: &v35.MetadataKey{ + Key: authzNamespace, + Path: []*v35.MetadataKey_PathSegment{ + &v35.MetadataKey_PathSegment{ + Segment: &v35.MetadataKey_PathSegment_Key{ + Key: "", + }, + }, + }, + }, + }, + }, + }, + }, + } + return []*routev3.RateLimit{&rateLimitForRequestTokenCount, &rateLimitForResponseTokenCount, &rateLimitForRequestCount} +} func generateRateLimitPolicy(ratelimitCriteria *ratelimitCriteria) []*routev3.RateLimit { environmentValue := ratelimitCriteria.environment @@ -526,9 +665,21 @@ func generateFilterConfigToSkipEnforcer() map[string]*anypb.Any { TypeUrl: extAuthzPerRouteName, Value: data, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } return map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, } } diff --git a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go index 85f673c38a..674f1c8086 100644 --- a/adapter/internal/oasparser/envoyconf/routes_with_clusters.go +++ b/adapter/internal/oasparser/envoyconf/routes_with_clusters.go @@ -39,6 +39,7 @@ import ( routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" cors_filter_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" extAuthService "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" + extProcessorv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_proc/v3" lua "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/lua/v3" tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" upstreams "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" @@ -859,13 +860,25 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } corsFilter, _ := anypb.New(corsPolicy) - perRouteFilterConfigs := map[string]*any.Any{ wellknown.HTTPExternalAuthorization: extAuthzFilter, LuaLocal: luaFilter, wellknown.CORS: corsFilter, } - + if !resource.GetEnableBackendBasedAIRatelimit() && !resource.GetEnableSubscriptionBasedAIRatelimit() { + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + perRouteFilterConfigs[HTTPExternalProcessor] = filterExtProc + } + logger.LoggerOasparser.Debugf("adding route : %s for API : %s", resourcePath, title) rateLimitPolicyLevel := "" @@ -916,6 +929,34 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } } routeConfig := resource.GetEndpoints().Config + metaData := &corev3.Metadata{} + if resource.GetEnableBackendBasedAIRatelimit() || resource.GetEnableSubscriptionBasedAIRatelimit() { + metaData = &corev3.Metadata{ + FilterMetadata: map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "EnableBackendBasedAIRatelimit": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: fmt.Sprintf("%t", resource.GetEnableBackendBasedAIRatelimit()), + }, + }, + "EnableSubscriptionBasedAIRatelimit": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: fmt.Sprintf("%t", resource.GetEnableSubscriptionBasedAIRatelimit()), + }, + }, + "BackendBasedAIRatelimitDescriptorValue": &structpb.Value{ + Kind: &structpb.Value_StringValue{ + StringValue: resource.GetBackendBasedAIRatelimitDescriptorValue(), + }, + }, + }, + }, + }, + } + } else { + metaData = nil + } if resource.HasPolicies() { logger.LoggerOasparser.Debug("Start creating routes for resource with policies") operations := resource.GetOperations() @@ -1050,12 +1091,12 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error metadataValue := operation.GetMethod() + "_to_" + newMethod match2.DynamicMetadata = generateMetadataMatcherForInternalRoutes(metadataValue) - action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) - action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) + action1 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) + action2 := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) // Create route1 for current method. // Do not add policies to route config. Send via enforcer - route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, nil, decorator, perRouteFilterConfigs, + route1 := generateRouteConfig(xWso2Basepath+operation.GetMethod(), match1, action1, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // Create route2 for new method. @@ -1066,7 +1107,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error action2.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } configToSkipEnforcer := generateFilterConfigToSkipEnforcer() - route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, nil, decorator, configToSkipEnforcer, + route2 := generateRouteConfig(xWso2Basepath, match2, action2, requestRedirectAction, metaData, decorator, configToSkipEnforcer, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route1) @@ -1074,7 +1115,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else { var action *routev3.Route_Route if requestRedirectAction == nil { - action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()]) + action = generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, mirrorClusterNames[operation.GetID()], resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) } logger.LoggerOasparser.Debug("Creating routes for resource with policies", resourcePath, operation.GetMethod()) // create route for current method. Add policies to route config. Send via enforcer @@ -1086,7 +1127,7 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } else if requestRedirectAction == nil { action.Route.RegexRewrite = generateRegexMatchAndSubstitute(routePath, resourcePath, pathMatchType) } - route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, nil, decorator, perRouteFilterConfigs, + route := generateRouteConfig(xWso2Basepath, match, action, requestRedirectAction, metaData, decorator, perRouteFilterConfigs, requestHeadersToAdd, requestHeadersToRemove, responseHeadersToAdd, responseHeadersToRemove) routes = append(routes, route) } @@ -1100,11 +1141,11 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error } match := generateRouteMatch(routePath) match.Headers = generateHTTPMethodMatcher(methodRegex, clusterName) - action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil) + action := generateRouteAction(apiType, routeConfig, rateLimitPolicyCriteria, nil, resource.GetEnableSubscriptionBasedAIRatelimit(), resource.GetEnableBackendBasedAIRatelimit(), resource.GetBackendBasedAIRatelimitDescriptorValue()) rewritePath := generateRoutePathForReWrite(basePath, resourcePath, pathMatchType) action.Route.RegexRewrite = generateRegexMatchAndSubstitute(rewritePath, resourcePath, pathMatchType) - - route := generateRouteConfig(xWso2Basepath, match, action, nil, nil, decorator, perRouteFilterConfigs, + + route := generateRouteConfig(xWso2Basepath, match, action, nil, metaData, decorator, perRouteFilterConfigs, nil, nil, nil, nil) // general headers to add and remove are included in this methods routes = append(routes, route) } @@ -1222,6 +1263,18 @@ func CreateAPIDefinitionRoute(basePath string, vHost string, methods []string, i }, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = routev3.Route{ Name: apiDefinitionQueryParam, Match: match, @@ -1230,6 +1283,7 @@ func CreateAPIDefinitionRoute(basePath string, vHost string, methods []string, i Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return &router @@ -1303,6 +1357,18 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v }, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = &routev3.Route{ Name: endpoint, //Categorize routes with same base path Match: match, @@ -1311,6 +1377,7 @@ func CreateAPIDefinitionEndpoint(adapterInternalAPI *model.AdapterInternalAPI, v Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return router @@ -1347,6 +1414,17 @@ func CreateHealthEndpoint() *routev3.Route { Value: data, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } router = routev3.Route{ Name: healthPath, //Categorize routes with same base path Match: match, @@ -1364,6 +1442,7 @@ func CreateHealthEndpoint() *routev3.Route { Decorator: decorator, TypedPerFilterConfig: map[string]*any.Any{ wellknown.HTTPExternalAuthorization: filter, + HTTPExternalProcessor : filterExtProc, }, } return &router @@ -1388,6 +1467,18 @@ func CreateReadyEndpoint() *routev3.Route { Operation: readyPath, } + perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{ + Override: &extProcessorv3.ExtProcPerRoute_Disabled{ + Disabled: true, + }, + } + + dataExtProc, _ := proto.Marshal(&perFilterConfigExtProc) + filterExtProc := &any.Any{ + TypeUrl: extProcPerRouteName, + Value: dataExtProc, + } + router = routev3.Route{ Name: readyPath, //Categorize routes with same base path Match: match, @@ -1400,6 +1491,9 @@ func CreateReadyEndpoint() *routev3.Route { }, Metadata: nil, Decorator: decorator, + TypedPerFilterConfig: map[string]*any.Any{ + HTTPExternalProcessor : filterExtProc, + }, } return &router } diff --git a/adapter/internal/oasparser/model/adapter_internal_api.go b/adapter/internal/oasparser/model/adapter_internal_api.go index 061f6c27ee..12a3bd4e53 100644 --- a/adapter/internal/oasparser/model/adapter_internal_api.go +++ b/adapter/internal/oasparser/model/adapter_internal_api.go @@ -447,7 +447,7 @@ func (adapterInternalAPI *AdapterInternalAPI) Validate() error { // SetInfoHTTPRouteCR populates resources and endpoints of adapterInternalAPI. httpRoute.Spec.Rules.Matches // are used to create resources and httpRoute.Spec.Rules.BackendRefs are used to create EndpointClusters. -func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams) error { +func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwapiv1.HTTPRoute, resourceParams ResourceParams, isAiSubscriptionRatelimitEnabled bool, ruleIdxToAiRatelimitPolicyMapping map[int]*dpv1alpha3.AIRateLimitPolicy) error { var resources []*Resource outputAuthScheme := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.AuthSchemes))) outputAPIPolicy := utils.TieBreaker(utils.GetPtrSlice(maps.Values(resourceParams.APIPolicies))) @@ -469,7 +469,7 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap ratelimitPolicy = *outputRatelimitPolicy } - for _, rule := range httpRoute.Spec.Rules { + for ruleID, rule := range httpRoute.Spec.Rules { var endPoints []Endpoint var policies = OperationPolicies{} var circuitBreaker *dpv1alpha2.CircuitBreaker @@ -491,6 +491,16 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap var securityConfig []EndpointSecurity var mirrorEndpointClusters []*EndpointCluster + enableBackendBasedAIRatelimit := false + descriptorValue := "" + if aiRatelimitPolicy, exists := ruleIdxToAiRatelimitPolicyMapping[ruleID]; exists { + loggers.LoggerAPI.Infof("Found AI ratelimit mapping for ruleId: %d, related api: %s", ruleID, adapterInternalAPI.UUID) + enableBackendBasedAIRatelimit = true + descriptorValue = prepareAIRatelimitIdentifier(adapterInternalAPI.OrganizationID, utils.NamespacedName(aiRatelimitPolicy), &aiRatelimitPolicy.Spec) + } else { + loggers.LoggerAPI.Infof("Could not find AIratelimit for ruleId: %d, len of map: %d, related api: %s", ruleID, len(ruleIdxToAiRatelimitPolicyMapping), adapterInternalAPI.UUID) + } + backendBasePath := "" for _, backend := range rule.BackendRefs { backendName := types.NamespacedName{ @@ -847,12 +857,17 @@ func (adapterInternalAPI *AdapterInternalAPI) SetInfoHTTPRouteCR(httpRoute *gwap operations := getAllowedOperations(match.Method, policies, apiAuth, parseRateLimitPolicyToInternal(resourceRatelimitPolicy), scopes, mirrorEndpointClusters) - resource := &Resource{path: resourcePath, + + resource := &Resource{ + path: resourcePath, methods: operations, pathMatchType: *match.Path.Type, hasPolicies: true, iD: uuid.New().String(), hasRequestRedirectFilter: hasRequestRedirectPolicy, + enableSubscriptionBasedAIRatelimit: isAiSubscriptionRatelimitEnabled, + enableBackendBasedAIRatelimit: enableBackendBasedAIRatelimit, + backendBasedAIRatelimitDescriptorValue: descriptorValue, } resource.endpoints = &EndpointCluster{ @@ -1301,3 +1316,11 @@ func CreateDummyAdapterInternalAPIForTests(title, version, basePath string, reso resources: resources, } } + +func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { + targetNamespace := string(namespacedName.Namespace) + if spec.TargetRef.Namespace != nil && string(*spec.TargetRef.Namespace) != "" { + targetNamespace = string(*spec.TargetRef.Namespace) + } + return fmt.Sprintf("%s-%s-%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name), targetNamespace, string(spec.TargetRef.Name)) +} diff --git a/adapter/internal/oasparser/model/resource.go b/adapter/internal/oasparser/model/resource.go index 9fafd7f875..a953e3a850 100644 --- a/adapter/internal/oasparser/model/resource.go +++ b/adapter/internal/oasparser/model/resource.go @@ -36,15 +36,18 @@ import ( // These values are populated from extensions/properties // mentioned under pathItem. type Resource struct { - path string - pathMatchType gwapiv1.PathMatchType - methods []*Operation - iD string - endpoints *EndpointCluster - endpointSecurity []*EndpointSecurity - vendorExtensions map[string]interface{} - hasPolicies bool - hasRequestRedirectFilter bool + path string + pathMatchType gwapiv1.PathMatchType + methods []*Operation + iD string + endpoints *EndpointCluster + endpointSecurity []*EndpointSecurity + vendorExtensions map[string]interface{} + hasPolicies bool + hasRequestRedirectFilter bool + enableSubscriptionBasedAIRatelimit bool + enableBackendBasedAIRatelimit bool + backendBasedAIRatelimitDescriptorValue string } // GetEndpointSecurity returns the endpoint security object of a given resource. @@ -189,3 +192,18 @@ func SortResources(resources []*Resource) []*Resource { sort.Sort(byPath(resources)) return resources } + +// GetEnableSubscriptionBasedAIRatelimit returns the value of enableSubscriptionBasedAIRatelimit. +func (resource *Resource) GetEnableSubscriptionBasedAIRatelimit() bool { + return resource.enableSubscriptionBasedAIRatelimit +} + +// GetEnableBackendBasedAIRatelimit returns the value of enableBackendBasedAIRatelimit. +func (resource *Resource) GetEnableBackendBasedAIRatelimit() bool { + return resource.enableBackendBasedAIRatelimit +} + +// GetBackendBasedAIRatelimitDescriptorValue returns the value of backendBasedAIRatelimitDescriptorValue. +func (resource *Resource) GetBackendBasedAIRatelimitDescriptorValue() string { + return resource.backendBasedAIRatelimitDescriptorValue +} \ No newline at end of file diff --git a/adapter/internal/operator/PROJECT b/adapter/internal/operator/PROJECT index 1c9a275a47..70447f6ad7 100644 --- a/adapter/internal/operator/PROJECT +++ b/adapter/internal/operator/PROJECT @@ -155,4 +155,13 @@ resources: kind: GQLRoute path: github.com/wso2/apk/adapter/internal/operator/apis/dp/v1alpha2 version: v1alpha2 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: wso2.com + group: dp + kind: AIRateLimitPolicy + path: github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3 + version: v1alpha3 version: "3" diff --git a/adapter/internal/operator/config/crd/kustomization.yaml b/adapter/internal/operator/config/crd/kustomization.yaml index f2eb17419e..5837a1fc4e 100644 --- a/adapter/internal/operator/config/crd/kustomization.yaml +++ b/adapter/internal/operator/config/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/cp.wso2.com_subscriptions.yaml - bases/dp.wso2.com_scopes.yaml - bases/dp.wso2.com_ratelimitpolicies.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml - bases/dp.wso2.com_backends.yaml - bases/dp.wso2.com_interceptorservices.yaml - bases/dp.wso2.com_jwtissuers.yaml diff --git a/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml b/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..2d28892f72 --- /dev/null +++ b/adapter/internal/operator/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: airatelimitpolicies.dp.wso2.com diff --git a/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml b/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b2cde3fe7d --- /dev/null +++ b/adapter/internal/operator/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: airatelimitpolicies.dp.wso2.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml new file mode 100644 index 0000000000..c6afbc67c6 --- /dev/null +++ b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-editor-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml new file mode 100644 index 0000000000..5fe04d1924 --- /dev/null +++ b/adapter/internal/operator/config/rbac/dp_airatelimitpolicy_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-viewer-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - get + - list + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/adapter/internal/operator/constants/constants.go b/adapter/internal/operator/constants/constants.go index 872c3491e3..cbafee9184 100644 --- a/adapter/internal/operator/constants/constants.go +++ b/adapter/internal/operator/constants/constants.go @@ -45,10 +45,11 @@ const ( KindAPI = "API" KindService = "Service" //TODO(amali) remove this after fixing the issue in https://github.com/wso2/apk/issues/383 - KindResource = "Resource" - KindScope = "Scope" - KindBackend = "Backend" - KindGateway = "Gateway" + KindResource = "Resource" + KindScope = "Scope" + KindBackend = "Backend" + KindGateway = "Gateway" + KindSubscription = "Subscription" ) // Env types diff --git a/adapter/internal/operator/controllers/dp/api_controller.go b/adapter/internal/operator/controllers/dp/api_controller.go index b96e7aa310..0ba0592549 100644 --- a/adapter/internal/operator/controllers/dp/api_controller.go +++ b/adapter/internal/operator/controllers/dp/api_controller.go @@ -58,8 +58,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" k8client "sigs.k8s.io/controller-runtime/pkg/client" + cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" - "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" corev1 "k8s.io/api/core/v1" @@ -82,20 +82,24 @@ const ( // apiAPIPolicyIndex Index for API level apipolicies apiAPIPolicyIndex = "apiAPIPolicyIndex" // apiAPIPolicyResourceIndex Index for resource level apipolicies - apiAPIPolicyResourceIndex = "apiAPIPolicyResourceIndex" - serviceHTTPRouteIndex = "serviceHTTPRouteIndex" - httprouteScopeIndex = "httprouteScopeIndex" - gqlRouteScopeIndex = "gqlRouteScopeIndex" - configMapBackend = "configMapBackend" - configMapAPIDefinition = "configMapAPIDefinition" - secretBackend = "secretBackend" - configMapAuthentication = "configMapAuthentication" - secretAuthentication = "secretAuthentication" - backendHTTPRouteIndex = "backendHTTPRouteIndex" - backendGQLRouteIndex = "backendGQLRouteIndex" - interceptorServiceAPIPolicyIndex = "interceptorServiceAPIPolicyIndex" - backendInterceptorServiceIndex = "backendInterceptorServiceIndex" - backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" + apiAPIPolicyResourceIndex = "apiAPIPolicyResourceIndex" + serviceHTTPRouteIndex = "serviceHTTPRouteIndex" + httprouteScopeIndex = "httprouteScopeIndex" + gqlRouteScopeIndex = "gqlRouteScopeIndex" + configMapBackend = "configMapBackend" + configMapAPIDefinition = "configMapAPIDefinition" + secretBackend = "secretBackend" + configMapAuthentication = "configMapAuthentication" + secretAuthentication = "secretAuthentication" + backendHTTPRouteIndex = "backendHTTPRouteIndex" + backendGQLRouteIndex = "backendGQLRouteIndex" + interceptorServiceAPIPolicyIndex = "interceptorServiceAPIPolicyIndex" + backendInterceptorServiceIndex = "backendInterceptorServiceIndex" + backendJWTAPIPolicyIndex = "backendJWTAPIPolicyIndex" + aiRatelimitPolicyToBackendIndex = "aiRatelimitPolicyToBackendIndex" + aiRatelimitPolicyToSubscriptionIndex = "aiRatelimitPolicyToSubscriptionIndex" + subscriptionToAPIIndex = "subscriptionToAPIIndex" + apiToSubscriptionIndex = "apiToSubscriptionIndex" ) var ( @@ -217,6 +221,18 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera return err } + if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForAIRatelimitPolicy), + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching AIRatelimitPolicy resources: %v", err)) + return err + } + + if err := c.Watch(source.Kind(mgr.GetCache(), &cpv1alpha2.Subscription{}), handler.EnqueueRequestsFromMapFunc(apiReconciler.populateAPIReconcileRequestsForSubscription), + predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2645, logging.BLOCKER, "Error watching Subscription resources: %v", err)) + return err + } + loggers.LoggerAPKOperator.Info("API Controller successfully started. Watching API Objects....") go apiReconciler.handleStatus() go apiReconciler.handleLabels(ctx) @@ -244,6 +260,9 @@ func NewAPIController(mgr manager.Manager, operatorDataStore *synchronizer.Opera // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies/status,verbs=get;update;patch // +kubebuilder:rbac:groups=dp.wso2.com,resources=ratelimitpolicies/finalizers,verbs=update +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -378,6 +397,7 @@ func (apiReconciler *APIReconciler) resolveAPIRefs(ctx context.Context, api dpv1 apiRef.String(), namespace, string(api.ObjectMeta.UID), "api definition file not found") } } + apiReconciler.resolveAiSubscriptionRatelimitPolicies(ctx, apiState) if len(apiState.Authentications) > 0 { if apiState.MutualSSL, err = apiReconciler.resolveAuthentications(ctx, apiState.Authentications); err != nil { return nil, fmt.Errorf("error while resolving authentication %v in namespace: %s was not found. %s", @@ -814,6 +834,29 @@ func (apiReconciler *APIReconciler) getAPIPolicyChildrenRefs(ctx context.Context return interceptorServices, backendJWTs, subscriptionValidation, nil } +func (apiReconciler *APIReconciler) resolveAiSubscriptionRatelimitPolicies(ctx context.Context, apiState *synchronizer.APIState) { + apiState.IsAiSubscriptionRatelimitEnabled = false + subscriptionList := &cpv1alpha2.SubscriptionList{} + if err := apiReconciler.client.List(ctx, subscriptionList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(subscriptionToAPIIndex, utils.GetSubscriptionToAPIIndexID(apiState.APIDefinition.Spec.APIName, apiState.APIDefinition.Spec.APIVersion)), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated subscription found for API: %s", utils.NamespacedName(apiState.APIDefinition)) + return + } + for _, subscription := range subscriptionList.Items { + aiRatelimitPolicyList := &dpv1alpha3.AIRateLimitPolicyList{} + if err := apiReconciler.client.List(ctx, aiRatelimitPolicyList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToSubscriptionIndex, utils.NamespacedName(&subscription).String()), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated aiRatelimitPolicy found for Subscription: %s", utils.NamespacedName(&subscription)) + return + } + if len(aiRatelimitPolicyList.Items) > 0 { + apiState.IsAiSubscriptionRatelimitEnabled = true + } + } +} + func (apiReconciler *APIReconciler) resolveAuthentications(ctx context.Context, authentications map[string]dpv1alpha2.Authentication) (*dpv1alpha2.MutualSSL, error) { resolvedMutualSSL := dpv1alpha2.MutualSSL{} @@ -833,12 +876,44 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte // Resolve backends in HTTPRoute httpRoute := httpRouteState.HTTPRouteCombined - for _, rule := range httpRoute.Spec.Rules { + // httpRouteState.AiRatelimit_HttpRouteRulesMapping = aiRatelimitPolicy_routeRulematching + // httpRouteState.AiRatelimitPolicyMapping = aiRatelimitPolicyMapping + ruleIdxToAiRatelimitPolicyMapping := make(map[int]*dpv1alpha3.AIRateLimitPolicy) + httpRouteState.RuleIdxToAiRatelimitPolicyMapping = ruleIdxToAiRatelimitPolicyMapping + for id, rule := range httpRoute.Spec.Rules { for _, backend := range rule.BackendRefs { backendNamespacedName := types.NamespacedName{ Name: string(backend.Name), Namespace: utils.GetNamespace(backend.Namespace, httpRoute.Namespace), } + aiRLPolicyList := &dpv1alpha3.AIRateLimitPolicyList{} + if err := apiReconciler.client.List(ctx, aiRLPolicyList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(aiRatelimitPolicyToBackendIndex, backendNamespacedName.String()), + }); err != nil { + loggers.LoggerAPKOperator.Infof("No associated AI ratelimit policy found for : %s", backendNamespacedName.String()) + } else { + for _, aiRLPolicy := range aiRLPolicyList.Items { + loggers.LoggerAPKOperator.Infof("Adding mapping for ruleid: %d to aiRLPolicy: %s", id, utils.NamespacedName(&aiRLPolicy)) + ruleIdxToAiRatelimitPolicyMapping[id] = &aiRLPolicy + // aiRlPolicyNN := utils.NamespacedName(&aiRLPolicy) + // if ruleList, exists := aiRatelimitPolicy_routeRulematching[aiRlPolicyNN]; !exists { + // ruleListNew := make([]*gwapiv1.HTTPRouteRule, 0) + // ruleListNew = append(ruleListNew, &rule) + // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleListNew + // } else { + // ruleList = append(ruleList, &rule) + // aiRatelimitPolicy_routeRulematching[aiRlPolicyNN] = ruleList + // } + // if aiRLList, exists := aiRatelimitPolicyMapping[aiRlPolicyNN]; !exists { + // aiRlListNew := make([]*v1alpha3.AIRateLimitPolicy, 0) + // aiRlListNew = append(aiRlListNew, &aiRLPolicy) + // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRlListNew + // } else { + // aiRLList = append(aiRLList, &aiRLPolicy) + // aiRatelimitPolicyMapping[aiRlPolicyNN] = aiRLList + // } + } + } if _, exists := backendMapping[backendNamespacedName.String()]; !exists { resolvedBackend := utils.GetResolvedBackend(ctx, apiReconciler.client, backendNamespacedName, &api) if resolvedBackend != nil { @@ -847,6 +922,7 @@ func (apiReconciler *APIReconciler) getResolvedBackendsMapping(ctx context.Conte return nil, fmt.Errorf("unable to find backend %s", backendNamespacedName.String()) } } + } for _, filter := range rule.Filters { @@ -926,6 +1002,22 @@ func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForSecret(ctx co return requests } +func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAIRatelimitPolicy(ctx context.Context, obj k8client.Object) []reconcile.Request { + requests := apiReconciler.getAPIsForAIRatelimitPolicy(ctx, obj) + if len(requests) > 0 { + apiReconciler.handleOwnerReference(ctx, obj, &requests) + } + return requests +} + +func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { + requests := apiReconciler.getAPIsForSubscription(ctx, obj) + // if len(requests) > 0 { + // apiReconciler.handleOwnerReference(ctx, obj, &requests) + // } + return requests +} + func (apiReconciler *APIReconciler) populateAPIReconcileRequestsForAuthentication(ctx context.Context, obj k8client.Object) []reconcile.Request { requests := apiReconciler.getAPIsForAuthentication(ctx, obj) if len(requests) > 0 { @@ -1343,6 +1435,60 @@ func (apiReconciler *APIReconciler) getAPIsForSecret(ctx context.Context, obj k8 return requests } +// getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected +// in AIRatelimitPolicy resources. +func (apiReconciler *APIReconciler) getAPIsForAIRatelimitPolicy(ctx context.Context, obj k8client.Object) []reconcile.Request { + aiRatelimitPolicy, ok := obj.(*dpv1alpha3.AIRateLimitPolicy) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) + return []reconcile.Request{} + } + + if aiRatelimitPolicy.Spec.TargetRef.Kind == constants.KindBackend { + backend := &dpv1alpha1.Backend{} + namespacedName := types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()), + } + + if err := apiReconciler.client.Get(ctx, namespacedName, backend); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2621, logging.MINOR, "Unable to find associated Backend for AIratelimitPolicy targetref: %s", namespacedName.String())) + return []reconcile.Request{} + } + return apiReconciler.getAPIsForBackend(ctx, backend) + } + return []reconcile.Request{} +} + +// getAPIsForAIRatelimitPolicy triggers the API controller reconcile method based on the changes detected +// in subscription resources. +func (apiReconciler *APIReconciler) getAPIsForSubscription(ctx context.Context, obj k8client.Object) []reconcile.Request { + subscription, ok := obj.(*cpv1alpha2.Subscription) + if !ok { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2622, logging.TRIVIAL, "Unexpected object type, bypassing reconciliation: %v", obj)) + return []reconcile.Request{} + } + apiList := &dpv1alpha2.APIList{} + if err := apiReconciler.client.List(ctx, apiList, &k8client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(apiToSubscriptionIndex, utils.GetSubscriptionToAPIIndexID(subscription.Spec.API.Name, subscription.Spec.API.Version)), + }); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2649, logging.CRITICAL, "Unable to find associated APIs for subscription: %s, error: %v", utils.NamespacedName(subscription).String(), err.Error())) + return []reconcile.Request{} + } + requests := []reconcile.Request{} + for _, api := range apiList.Items { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: api.Name, + Namespace: api.Namespace}, + } + requests = append(requests, req) + loggers.LoggerAPKOperator.Infof("Adding reconcile request for API: %s/%s with API UUID: %v due to change in subscription: %v", api.Namespace, api.Name, + string(api.ObjectMeta.UID), utils.NamespacedName(subscription).String()) + } + return requests +} + // getAPIForAuthentication triggers the API controller reconcile method based on the changes detected // from Authentication objects. If the changes are done for an API stored in the Operator Data store, // a new reconcile event will be created and added to the reconcile event queue. @@ -1873,6 +2019,60 @@ func addIndexes(ctx context.Context, mgr manager.Manager) error { return err } + // AIRatelimitPolicy to Backend indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToBackendIndex, + func(rawObj k8client.Object) []string { + aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) + var backends []string + namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) + backends = append(backends, types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: namespace, + }.String()) + return backends + }); err != nil { + return err + } + + // AIRatelimitPolicy to Subscription indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha3.AIRateLimitPolicy{}, aiRatelimitPolicyToSubscriptionIndex, + func(rawObj k8client.Object) []string { + aiRatelimitPolicy := rawObj.(*dpv1alpha3.AIRateLimitPolicy) + var subscriptions []string + namespace := utils.GetNamespace(aiRatelimitPolicy.Spec.TargetRef.Namespace, aiRatelimitPolicy.GetNamespace()) + subscriptions = append(subscriptions, types.NamespacedName{ + Name: string(aiRatelimitPolicy.Spec.TargetRef.Name), + Namespace: namespace, + }.String()) + return subscriptions + }); err != nil { + return err + } + + // Subscription to API indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &cpv1alpha2.Subscription{}, subscriptionToAPIIndex, + func(rawObj k8client.Object) []string { + subscription := rawObj.(*cpv1alpha2.Subscription) + var subscriptions []string + subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", subscription.Spec.API.Name, subscription.Spec.API.Version) + subscriptions = append(subscriptions, subscriptionIdentifierForIndex) + return subscriptions + }); err != nil { + return err + } + + // API to Subscription indexer + if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.API{}, apiToSubscriptionIndex, + func(rawObj k8client.Object) []string { + api := rawObj.(*dpv1alpha2.API) + var apis []string + subscriptionIdentifierForIndex := fmt.Sprintf("%s_%s", api.Spec.APIName, api.Spec.APIVersion) + apis = append(apis, subscriptionIdentifierForIndex) + return apis + }); err != nil { + return err + } + // authentication to API indexer if err := mgr.GetFieldIndexer().IndexField(ctx, &dpv1alpha2.Authentication{}, apiAuthenticationIndex, func(rawObj k8client.Object) []string { @@ -2608,7 +2808,7 @@ func findProdSandEndpoints(apiState *synchronizer.APIState) (string, string, str } func pickOneCorsForCP(apiState *synchronizer.APIState) *controlplane.CORSPolicy { - apiPolicies := []v1alpha2.APIPolicy{} + apiPolicies := []dpv1alpha2.APIPolicy{} for _, apiPolicy := range apiState.APIPolicies { apiPolicies = append(apiPolicies, apiPolicy) } @@ -2616,7 +2816,7 @@ func pickOneCorsForCP(apiState *synchronizer.APIState) *controlplane.CORSPolicy apiPolicies = append(apiPolicies, apiPolicy) } for _, apiPolicy := range apiPolicies { - corsPolicy := v1alpha2.CORSPolicy{} + corsPolicy := dpv1alpha2.CORSPolicy{} found := false if apiPolicy.Spec.Override != nil && apiPolicy.Spec.Override.CORSPolicy != nil { corsPolicy = *apiPolicy.Spec.Override.CORSPolicy @@ -2676,14 +2876,14 @@ func geSandVhost(apiState *synchronizer.APIState) string { } func prepareSecuritySchemeForCP(apiState *synchronizer.APIState) ([]string, string) { - var pickedAuth *v1alpha2.Authentication + var pickedAuth *dpv1alpha2.Authentication authHeader := "Authorization" for _, auth := range apiState.Authentications { pickedAuth = &auth break } if pickedAuth != nil { - var authSpec *v1alpha2.AuthSpec + var authSpec *dpv1alpha2.AuthSpec if pickedAuth.Spec.Override != nil { authSpec = pickedAuth.Spec.Override } else { diff --git a/adapter/internal/operator/controllers/dp/suite_test.go b/adapter/internal/operator/controllers/dp/suite_test.go index 017df70bcb..3dbdd04c94 100644 --- a/adapter/internal/operator/controllers/dp/suite_test.go +++ b/adapter/internal/operator/controllers/dp/suite_test.go @@ -33,6 +33,7 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" //+kubebuilder:scaffold:imports ) @@ -70,6 +71,9 @@ var _ = BeforeSuite(func() { err = dpv1alpha2.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = dpv1alpha3.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/adapter/internal/operator/operator.go b/adapter/internal/operator/operator.go index 8558432053..ece0e47e1a 100644 --- a/adapter/internal/operator/operator.go +++ b/adapter/internal/operator/operator.go @@ -50,6 +50,7 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + cpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/cp/v1alpha2" //+kubebuilder:scaffold:imports ) @@ -67,7 +68,10 @@ func init() { utilruntime.Must(gwapiv1a2.AddToScheme(scheme)) utilruntime.Must(dpv1alpha2.AddToScheme(scheme)) + utilruntime.Must(dpv1alpha3.AddToScheme(scheme)) + + utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/adapter/internal/operator/synchronizer/api_state.go b/adapter/internal/operator/synchronizer/api_state.go index e6ce50560b..2fdc9e34de 100644 --- a/adapter/internal/operator/synchronizer/api_state.go +++ b/adapter/internal/operator/synchronizer/api_state.go @@ -44,6 +44,7 @@ type APIState struct { APIDefinitionFile []byte SubscriptionValidation bool MutualSSL *v1alpha2.MutualSSL + IsAiSubscriptionRatelimitEnabled bool } // HTTPRouteState holds the state of the deployed httpRoutes. This state is compared with @@ -54,6 +55,7 @@ type HTTPRouteState struct { HTTPRoutePartitions map[string]*gwapiv1.HTTPRoute BackendMapping map[string]*v1alpha2.ResolvedBackend Scopes map[string]v1alpha1.Scope + RuleIdxToAiRatelimitPolicyMapping map[int]*v1alpha3.AIRateLimitPolicy } // GQLRouteState holds the state of the deployed gqlRoutes. This state is compared with diff --git a/adapter/internal/operator/synchronizer/data_store.go b/adapter/internal/operator/synchronizer/data_store.go index 0c861d315b..1ecb67f1c2 100644 --- a/adapter/internal/operator/synchronizer/data_store.go +++ b/adapter/internal/operator/synchronizer/data_store.go @@ -90,7 +90,28 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced "Production"); routesUpdated { updated = true events = append(events, routeEvents...) + } else { + // Check whether AIRatelimitPolicy is updated + for key, aiRl := range apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { + if cachedAIRl, exists := cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { + if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { + loggers.LoggerAPI.Infof("Returning true * %s %s %d %d", utils.NamespacedName(cachedAIRl).String(), utils.NamespacedName(aiRl).String(), cachedAIRl.Generation, aiRl.Generation) + updated = true + break + } + } else { + loggers.LoggerAPI.Info("Returning true&&") + updated = true + break + } + } + if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { + loggers.LoggerAPI.Info("Returning true ***") + updated = true + } } + cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping = apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping + } else { if cachedAPI.ProdHTTPRoute != nil { updated = true @@ -98,6 +119,7 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } cachedAPI.ProdHTTPRoute = nil } + if apiState.ProdGQLRoute != nil { if cachedAPI.ProdGQLRoute == nil { cachedAPI.ProdGQLRoute = apiState.ProdGQLRoute @@ -123,7 +145,27 @@ func (ods *OperatorDataStore) processAPIState(apiNamespacedName types.Namespaced } else if routeEvents, routesUpdated := updateHTTPRoute(apiState.SandHTTPRoute, cachedAPI.SandHTTPRoute, "Sandbox"); routesUpdated { updated = true events = append(events, routeEvents...) + } else { + // Check whether AIRatelimitPolicy is updated + for key, aiRl := range apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping { + if cachedAIRl, exists := cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping[key]; exists { + if utils.NamespacedName(cachedAIRl).String() != utils.NamespacedName(aiRl).String() || cachedAIRl.Generation != aiRl.Generation { + loggers.LoggerAPI.Info("Returning true") + updated = true + break + } + } else { + loggers.LoggerAPI.Info("Returning true") + updated = true + break + } + } + if len(cachedAPI.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) != len(apiState.ProdHTTPRoute.RuleIdxToAiRatelimitPolicyMapping) { + loggers.LoggerAPI.Info("Returning true") + updated = true + } } + cachedAPI.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping = apiState.SandHTTPRoute.RuleIdxToAiRatelimitPolicyMapping } else { if cachedAPI.SandHTTPRoute != nil { updated = true diff --git a/adapter/internal/operator/synchronizer/rest_api.go b/adapter/internal/operator/synchronizer/rest_api.go index d16be379c2..eb872679dc 100644 --- a/adapter/internal/operator/synchronizer/rest_api.go +++ b/adapter/internal/operator/synchronizer/rest_api.go @@ -78,7 +78,7 @@ func UpdateInternalMapsFromHTTPRoute(apiState APIState, httpRoute *HTTPRouteStat } // generateAdapterInternalAPI this will populate a AdapterInternalAPI representation for an HTTPRoute -func generateAdapterInternalAPI(apiState APIState, httpRoute *HTTPRouteState, envType string) (*model.AdapterInternalAPI, error) { +func generateAdapterInternalAPI(apiState APIState, httpRouteState *HTTPRouteState, envType string) (*model.AdapterInternalAPI, error) { var adapterInternalAPI model.AdapterInternalAPI adapterInternalAPI.SetIsDefaultVersion(apiState.APIDefinition.Spec.IsDefaultVersion) adapterInternalAPI.SetInfoAPICR(*apiState.APIDefinition) @@ -98,16 +98,16 @@ func generateAdapterInternalAPI(apiState APIState, httpRoute *HTTPRouteState, en resourceParams := model.ResourceParams{ AuthSchemes: apiState.Authentications, ResourceAuthSchemes: apiState.ResourceAuthentications, - BackendMapping: httpRoute.BackendMapping, + BackendMapping: httpRouteState.BackendMapping, APIPolicies: apiState.APIPolicies, ResourceAPIPolicies: apiState.ResourceAPIPolicies, - ResourceScopes: httpRoute.Scopes, + ResourceScopes: httpRouteState.Scopes, InterceptorServiceMapping: apiState.InterceptorServiceMapping, BackendJWTMapping: apiState.BackendJWTMapping, RateLimitPolicies: apiState.RateLimitPolicies, ResourceRateLimitPolicies: apiState.ResourceRateLimitPolicies, } - if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRoute.HTTPRouteCombined, resourceParams); err != nil { + if err := adapterInternalAPI.SetInfoHTTPRouteCR(httpRouteState.HTTPRouteCombined, resourceParams, apiState.IsAiSubscriptionRatelimitEnabled, httpRouteState.RuleIdxToAiRatelimitPolicyMapping); err != nil { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2631, logging.MAJOR, "Error setting HttpRoute CR info to adapterInternalAPI. %v", err)) return nil, err } diff --git a/adapter/internal/operator/utils/utils.go b/adapter/internal/operator/utils/utils.go index 064c647f0b..b9eb1020cf 100644 --- a/adapter/internal/operator/utils/utils.go +++ b/adapter/internal/operator/utils/utils.go @@ -670,3 +670,8 @@ func ContainsString(list []string, target string) bool { } return false } + +// GetSubscriptionToAPIIndexID returns the id which can be used to list subscriptions related to a api. +func GetSubscriptionToAPIIndexID(name string, version string) string { + return fmt.Sprintf("%s_%s", name, version) +} diff --git a/common-controller/go.mod b/common-controller/go.mod index 5e0e92001e..d5707c7a84 100644 --- a/common-controller/go.mod +++ b/common-controller/go.mod @@ -16,14 +16,14 @@ require ( ) require ( - github.com/envoyproxy/go-control-plane v0.12.0 + github.com/envoyproxy/go-control-plane v0.13.0 github.com/gin-gonic/gin v1.9.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/pelletier/go-toml v1.9.5 github.com/redis/go-redis/v9 v9.2.1 github.com/wso2/apk/adapter v0.0.0-20231214082511-af2c8b8a19f1 github.com/wso2/apk/common-go-libs v0.0.0-20240304050809-a382bc6b0d82 - google.golang.org/grpc v1.62.0 + google.golang.org/grpc v1.65.0 ) replace github.com/wso2/apk/adapter => ../adapter @@ -31,6 +31,7 @@ replace github.com/wso2/apk/adapter => ../adapter replace github.com/wso2/apk/common-go-libs => ../common-go-libs require ( + cel.dev/expr v0.15.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -52,6 +53,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect @@ -63,16 +65,16 @@ require ( github.com/vektah/gqlparser v1.3.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -86,7 +88,7 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect @@ -99,24 +101,22 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/protobuf v1.34.1 gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/common-controller/go.sum b/common-controller/go.sum index 44ab8d7761..e3caf33f31 100644 --- a/common-controller/go.sum +++ b/common-controller/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -19,8 +21,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -29,8 +31,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -45,8 +47,8 @@ github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= -github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= @@ -100,14 +102,11 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -179,6 +178,8 @@ github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNc github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -187,8 +188,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:Om github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= @@ -225,8 +226,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -239,7 +241,6 @@ github.com/vektah/gqlparser v1.3.1 h1:8b0IcD3qZKWJQHSzynbDlrtP3IxVydZ2DZepCGofqf github.com/vektah/gqlparser v1.3.1/go.mod h1:bkVf0FX+Stjg/MHnm8mEyubuaArhNEqfQhF+OTiAL74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -257,9 +258,8 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8= golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= @@ -268,7 +268,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -277,21 +276,18 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -299,28 +295,21 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -332,7 +321,6 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -343,26 +331,20 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/common-controller/internal/cache/datastore.go b/common-controller/internal/cache/datastore.go index d9b71480b1..293a3c9e3c 100644 --- a/common-controller/internal/cache/datastore.go +++ b/common-controller/internal/cache/datastore.go @@ -33,6 +33,7 @@ type RatelimitDataStore struct { resolveSubscriptionRatelimitStore map[types.NamespacedName]dpv1alpha3.ResolveSubscriptionRatelimitPolicy customRatelimitStore map[types.NamespacedName]*dpv1alpha1.CustomRateLimitPolicyDef mu sync.Mutex + aiRatelimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec } // CreateNewOperatorDataStore creates a new RatelimitDataStore. @@ -89,6 +90,18 @@ func (ods *RatelimitDataStore) AddorUpdateCustomRatelimitToStore(rateLimit types ods.customRatelimitStore[rateLimit] = &customRateLimitPolicy } +// AddorUpdateAIRatelimitToStore adds a new ratelimit to the RatelimitDataStore. +func (ods *RatelimitDataStore) AddorUpdateAIRatelimitToStore(rateLimit types.NamespacedName, + aiRatelimitSpec dpv1alpha3.AIRateLimitPolicySpec) { + ods.mu.Lock() + defer ods.mu.Unlock() + logger.Infof("Adding/Updating AI ratelimit spec to cache") + if ods.aiRatelimitPolicySpecs == nil { + ods.aiRatelimitPolicySpecs = make(map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) + } + ods.aiRatelimitPolicySpecs[rateLimit] = &aiRatelimitSpec +} + // GetResolveRatelimitPolicy get cached ratelimit func (ods *RatelimitDataStore) GetResolveRatelimitPolicy(rateLimit types.NamespacedName) ([]dpv1alpha1.ResolveRateLimitAPIPolicy, bool) { var rateLimitPolicy []dpv1alpha1.ResolveRateLimitAPIPolicy @@ -109,6 +122,11 @@ func (ods *RatelimitDataStore) GetCachedCustomRatelimitPolicy(rateLimit types.Na return rateLimitPolicy, false } +// GetAIRatelimitPolicySpecs gets all the AIRatelimitPolicy stored in ods +func (ods *RatelimitDataStore) GetAIRatelimitPolicySpecs() map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec { + return ods.aiRatelimitPolicySpecs +} + // DeleteResolveRatelimitPolicy delete from ratelimit cache func (ods *RatelimitDataStore) DeleteResolveRatelimitPolicy(rateLimit types.NamespacedName) { ods.mu.Lock() @@ -125,6 +143,14 @@ func (ods *RatelimitDataStore) DeleteCachedCustomRatelimitPolicy(rateLimit types delete(ods.customRatelimitStore, rateLimit) } +// DeleteAIRatelimitPolicySpec delete from ratelimit cache +func (ods *RatelimitDataStore) DeleteAIRatelimitPolicySpec(rateLimit types.NamespacedName) { + ods.mu.Lock() + defer ods.mu.Unlock() + logger.Debug("Deleting AI ratelimit from cache") + delete(ods.aiRatelimitPolicySpecs, rateLimit) +} + // NamespacedName generates namespaced name for Kubernetes objects func NamespacedName(obj client.Object) types.NamespacedName { return types.NamespacedName{ diff --git a/common-controller/internal/operator/config/crd/kustomization.yaml b/common-controller/internal/operator/config/crd/kustomization.yaml index 169d615e98..f635c32349 100644 --- a/common-controller/internal/operator/config/crd/kustomization.yaml +++ b/common-controller/internal/operator/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/dp.wso2.com_ratelimitpolicies.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml - bases/dp.wso2.com_apis.yaml - bases/cp.wso2.com_applications.yaml - bases/cp.wso2.com_subscriptions.yaml diff --git a/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go new file mode 100644 index 0000000000..afe263ae8a --- /dev/null +++ b/common-controller/internal/operator/controllers/dp/airatelimitpolicy_controller.go @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dp + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + // "context" + // "fmt" + // "time" + + // logger "github.com/sirupsen/logrus" + // k8error "k8s.io/apimachinery/pkg/api/errors" + // "k8s.io/apimachinery/pkg/fields" + // "k8s.io/apimachinery/pkg/runtime" + // "k8s.io/apimachinery/pkg/types" + // ctrl "sigs.k8s.io/controller-runtime" + // "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + // "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + // "sigs.k8s.io/controller-runtime/pkg/reconcile" + + // k8client "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/source" + // gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/wso2/apk/adapter/pkg/logging" + cache "github.com/wso2/apk/common-controller/internal/cache" + "github.com/wso2/apk/common-controller/internal/config" + loggers "github.com/wso2/apk/common-controller/internal/loggers" + "github.com/wso2/apk/common-controller/internal/utils" + xds "github.com/wso2/apk/common-controller/internal/xds" + // dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" + // dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + "github.com/wso2/apk/common-go-libs/constants" +) + +// AIRateLimitPolicyReconciler reconciles a AIRateLimitPolicy object +type AIRateLimitPolicyReconciler struct { + client.Client + Scheme *runtime.Scheme + ods *cache.RatelimitDataStore +} + +// NewAIRatelimitController creates a new ratelimitcontroller instance. +func NewAIRatelimitController(mgr manager.Manager, ratelimitStore *cache.RatelimitDataStore) error { + aiRateLimitPolicyReconciler := &AIRateLimitPolicyReconciler{ + Client: mgr.GetClient(), + ods: ratelimitStore, + } + + // ctx := context.Background() + // if err := addIndexes(ctx, mgr); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2612, logging.BLOCKER, "Error adding indexes: %v", err)) + // return err + // } + + c, err := controller.New(constants.AIRatelimitController, mgr, controller.Options{Reconciler: aiRateLimitPolicyReconciler}) + if err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2663, logging.BLOCKER, + "Error creating Ratelimit controller: %v", err.Error())) + return err + } + + conf := config.ReadConfigs() + predicates := []predicate.Predicate{predicate.NewPredicateFuncs(utils.FilterByNamespaces(conf.CommonController.Operator.Namespaces))} + + // if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha2.API{}), + // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForAPI), predicates...); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2611, logging.BLOCKER, + // "Error watching API resources: %v", err)) + // return err + // } + + // if err := c.Watch(source.Kind(mgr.GetCache(), &gwapiv1.HTTPRoute{}), + // handler.EnqueueRequestsFromMapFunc(aiRateLimitPolicyReconciler.getRatelimitForHTTPRoute), predicates...); err != nil { + // loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2613, logging.BLOCKER, + // "Error watching HTTPRoute resources: %v", err)) + // return err + // } + + if err := c.Watch(source.Kind(mgr.GetCache(), &dpv1alpha3.AIRateLimitPolicy{}), &handler.EnqueueRequestForObject{}, predicates...); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error2639, logging.BLOCKER, + "Error watching Ratelimit resources: %v", err.Error())) + return err + } + + loggers.LoggerAPKOperator.Debug("RatelimitPolicy Controller successfully started. Watching RatelimitPolicy Objects...") + return nil +} + +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dp.wso2.com,resources=airatelimitpolicies/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the AIRateLimitPolicy object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.4/pkg/reconcile +func (r *AIRateLimitPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile...") + // TODO(user): your logic here + ratelimitKey := req.NamespacedName + var ratelimitPolicy dpv1alpha3.AIRateLimitPolicy + conf := config.ReadConfigs() + + // Check k8s RatelimitPolicy Availbility + if err := r.Client.Get(ctx, ratelimitKey, &ratelimitPolicy); err != nil { + loggers.LoggerAPKOperator.Errorf("Error retrieving AIRatelimit") + // It could be deletion event. So lets try to delete the related entried from the ods and update xds + r.ods.DeleteAIRatelimitPolicySpec(ratelimitKey) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } else { + loggers.LoggerAPKOperator.Infof("ratelimits found") + r.ods.AddorUpdateAIRatelimitToStore(ratelimitKey, ratelimitPolicy.Spec) + xds.UpdateRateLimitXDSCacheForAIRatelimitPolicies(r.ods.GetAIRatelimitPolicySpecs()) + xds.UpdateRateLimiterPolicies(conf.CommonController.Server.Label) + } + loggers.LoggerAPKOperator.Infof("AIRatelimit reconcile..*****.") + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *AIRateLimitPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&dpv1alpha3.AIRateLimitPolicy{}). + Complete(r) +} diff --git a/common-controller/internal/operator/controllers/dp/suite_test.go b/common-controller/internal/operator/controllers/dp/suite_test.go index fd1ec07c35..2cb4598924 100644 --- a/common-controller/internal/operator/controllers/dp/suite_test.go +++ b/common-controller/internal/operator/controllers/dp/suite_test.go @@ -31,6 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" + dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" //+kubebuilder:scaffold:imports ) @@ -65,6 +67,12 @@ var _ = BeforeSuite(func() { err = dpv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = dpv1alpha2.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = dpv1alpha3.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) diff --git a/common-controller/internal/operator/operator.go b/common-controller/internal/operator/operator.go index d28717211a..9bf0f9ad6a 100644 --- a/common-controller/internal/operator/operator.go +++ b/common-controller/internal/operator/operator.go @@ -63,6 +63,7 @@ func init() { utilruntime.Must(gwapiv1.AddToScheme(scheme)) utilruntime.Must(dpv1alpha1.AddToScheme(scheme)) utilruntime.Must(dpv1alpha2.AddToScheme(scheme)) + utilruntime.Must(dpv1alpha3.AddToScheme(scheme)) utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) utilruntime.Must(cpv1alpha2.AddToScheme(scheme)) utilruntime.Must(cpv1alpha3.AddToScheme(scheme)) @@ -164,6 +165,10 @@ func InitOperator(metricsConfig config.Metrics) { loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3114, logging.MAJOR, "Error creating JWT Issuer controller, error: %v", err)) } + if err := dpcontrollers.NewAIRatelimitController(mgr, ratelimitStore); err != nil { + loggers.LoggerAPKOperator.ErrorC(logging.PrintError(logging.Error3114, logging.MAJOR, + "Error creating JWT Issuer controller, error: %v", err)) + } config := config.ReadConfigs() if !(config.CommonController.ControlPlane.Enabled && config.CommonController.ControlPlane.Persistence.Type == "DB") { diff --git a/common-controller/internal/xds/ratelimiter_cache.go b/common-controller/internal/xds/ratelimiter_cache.go index 15a191a408..9b9d4bb089 100644 --- a/common-controller/internal/xds/ratelimiter_cache.go +++ b/common-controller/internal/xds/ratelimiter_cache.go @@ -33,20 +33,25 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" "github.com/wso2/apk/common-go-libs/constants" + "k8s.io/apimachinery/pkg/types" ) // Constants relevant to the route related ratelimit configurations const ( - DescriptorKeyForOrg = "org" - OrgMetadataKey = "customorg" - DescriptorKeyForEnvironment = "environment" - DescriptorKeyForPath = "path" - DescriptorKeyForMethod = "method" - DescriptorValueForAPIMethod = "ALL" - DescriptorValueForOperationMethod = ":method" - MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" - MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" - apiDefinitionClusterName = "api_definition_cluster" + DescriptorKeyForOrg = "org" + OrgMetadataKey = "customorg" + DescriptorKeyForEnvironment = "environment" + DescriptorKeyForPath = "path" + DescriptorKeyForMethod = "method" + DescriptorValueForAPIMethod = "ALL" + DescriptorValueForOperationMethod = ":method" + MetadataNamespaceForCustomPolicies = "apk.ratelimit.metadata" + MetadataNamespaceForWSO2Policies = "envoy.filters.http.ext_authz" + apiDefinitionClusterName = "api_definition_cluster" + DescriptorKeyForAIRequestTokenCount = "airequesttokencount" + DescriptorKeyForAIResponseTokenCount = "airesponsetokencount" + DescriptorKeyForAITotalTokenCount = "aitotaltokencount" + DescriptorKeyForAIRequestCount = "airequestcount" ) const ( @@ -81,6 +86,8 @@ type rateLimitPolicyCache struct { // org -> Custom Rate Limit Configs customRateLimitPolicies map[string]map[string]*rls_config.RateLimitDescriptor + aiRatelimitDescriptors []*rls_config.RateLimitDescriptor + // mutex for API level apiLevelMu sync.RWMutex @@ -282,6 +289,8 @@ func (r *rateLimitPolicyCache) generateRateLimitConfig() *rls_config.RateLimitCo } orgDescriptors = append(orgDescriptors, metadataDescriptors...) + // Add AI ratelimit descriptors + orgDescriptors = append(orgDescriptors, r.aiRatelimitDescriptors...) return &rls_config.RateLimitConfig{ Name: RateLimiterDomain, Domain: RateLimiterDomain, @@ -313,6 +322,61 @@ func (r *rateLimitPolicyCache) AddCustomRateLimitPolicies(customRateLimitPolicy } } +// ProcessAIratelimitPolicySpecsAndUpdateCache process the specs and update the cache +func (r *rateLimitPolicyCache) ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRateLimitPolicySpecs map[types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + aiRlDescriptors := make([]*rls_config.RateLimitDescriptor, 0) + loggers.LoggerAPKOperator.Infof("222222") + for namespacedName, spec := range aiRateLimitPolicySpecs { + logger.Infof("Adding : %s, %s", DescriptorKeyForAIRequestCount, prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec)) + logger.Infof("For airl: %s", namespacedName) + // Add descriptor for RequestTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIRequestTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.RequestTokenCount), + }, + }) + // Add descriptor for ResponseTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIResponseTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.ResponseTokenCount), + }, + }) + // Add descriptor for TotalTokenCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAITotalTokenCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.TokenCount.Unit), + RequestsPerUnit: uint32(spec.Override.TokenCount.TotalTokenCount), + }, + }) + // Add descriptor for RequestCount + aiRlDescriptors = append(aiRlDescriptors, &rls_config.RateLimitDescriptor{ + Key: DescriptorKeyForAIRequestCount, + Value: prepareAIRatelimitIdentifier(spec.Override.Organization, namespacedName, spec), + RateLimit: &rls_config.RateLimitPolicy{ + Unit: getRateLimitUnit(spec.Override.RequestCount.Unit), + RequestsPerUnit: uint32(spec.Override.RequestCount.RequestsPerUnit), + }, + }) + } + r.aiRatelimitDescriptors = aiRlDescriptors +} + +func prepareAIRatelimitIdentifier(org string, namespacedName types.NamespacedName, spec *dpv1alpha3.AIRateLimitPolicySpec) string { + targetNamespace := string(namespacedName.Namespace) + if spec.TargetRef.Namespace != nil && string(*spec.TargetRef.Namespace) != "" { + targetNamespace = string(*spec.TargetRef.Namespace) + } + return fmt.Sprintf("%s-%s-%s-%s-%s", org, string(namespacedName.Namespace), string(namespacedName.Name), targetNamespace, string(spec.TargetRef.Name)) +} + func (r *rateLimitPolicyCache) updateXdsCache(label string) bool { rlsConf := r.generateRateLimitConfig() version := fmt.Sprint(rand.Int(rand.Reader, maxRandomBigInt())) diff --git a/common-controller/internal/xds/server.go b/common-controller/internal/xds/server.go index 24be21d0ad..457f754d42 100644 --- a/common-controller/internal/xds/server.go +++ b/common-controller/internal/xds/server.go @@ -33,8 +33,10 @@ import ( wso2_cache "github.com/wso2/apk/adapter/pkg/discovery/protocol/cache/v3" eventhubTypes "github.com/wso2/apk/adapter/pkg/eventhub/types" + "github.com/wso2/apk/common-controller/internal/loggers" dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha3 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3" + apimachiner_types "k8s.io/apimachinery/pkg/types" ) // EnvoyInternalAPI struct use to hold envoy resources and adapter internal resources @@ -163,6 +165,15 @@ func UpdateRateLimitXDSCacheForCustomPolicies(customRateLimitPolicies dpv1alpha1 } } +// UpdateRateLimitXDSCacheForAIRatelimitPolicies updates the xDS cache of the RateLimiter for AI ratelimit policies. +func UpdateRateLimitXDSCacheForAIRatelimitPolicies(aiRatelimitPolicySpecs map[apimachiner_types.NamespacedName]*dpv1alpha3.AIRateLimitPolicySpec) { + loggers.LoggerAPKOperator.Infof("000000") + if len(aiRatelimitPolicySpecs) != 0 { + loggers.LoggerAPKOperator.Infof("1111111") + rlsPolicyCache.ProcessAIRatelimitPolicySpecsAndUpdateCache(aiRatelimitPolicySpecs) + } +} + // DeleteAPILevelRateLimitPolicies delete the ratelimit xds cache func DeleteAPILevelRateLimitPolicies(resolveRatelimitPolicyList []dpv1alpha1.ResolveRateLimitAPIPolicy) { diff --git a/common-go-libs/PROJECT b/common-go-libs/PROJECT index 51d234c0e9..5aee973d41 100644 --- a/common-go-libs/PROJECT +++ b/common-go-libs/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: wso2.com layout: - go.kubebuilder.io/v3 @@ -176,4 +180,13 @@ resources: kind: Subscription path: github.com/wso2/apk/common-go-libs/apis/cp/v1alpha3 version: v1alpha3 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: wso2.com + group: dp + kind: AIRateLimitPolicy + path: github.com/wso2/apk/common-go-libs/apis/dp/v1alpha3 + version: v1alpha3 version: "3" diff --git a/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go b/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go new file mode 100644 index 0000000000..200c4ee9bc --- /dev/null +++ b/common-go-libs/apis/dp/v1alpha3/airatelimitpolicy_types.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy +type AIRateLimitPolicySpec struct { + Override *AIRateLimit `json:"override,omitempty"` + Default *AIRateLimit `json:"default,omitempty"` + TargetRef gwapiv1b1.PolicyTargetReference `json:"targetRef,omitempty"` +} + +// AIRateLimit defines the AI ratelimit configuration +type AIRateLimit struct { + Organization string `json:"organization,omitempty"` + TokenCount *TokenCount `json:"tokenCount,omitempty"` + RequestCount *RequestCount `json:"requestCount,omitempty"` +} + +// TokenCount defines the Token based ratelimit configuration +type TokenCount struct { + // Unit is the unit of the requestsPerUnit + // + // +kubebuilder:validation:Enum=Minute;Hour;Day + Unit string `json:"unit,omitempty"` + + // RequestTokenCount specifies the maximum number of tokens allowed + // in AI requests within a given unit of time. This value limits the + // token count sent by the client to the AI service over the defined period. + // + // +kubebuilder:validation:Minimum=1 + RequestTokenCount uint32 `json:"requestTokenCount,omitempty"` + + // ResponseTokenCount specifies the maximum number of tokens allowed + // in AI responses within a given unit of time. This value limits the + // token count received by the client from the AI service over the defined period. + // + // +kubebuilder:validation:Minimum=1 + ResponseTokenCount uint32 `json:"responseTokenCount,omitempty"` + + // TotalTokenCount represents the maximum allowable total token count + // for both AI requests and responses within a specified unit of time. + // This value sets the limit for the number of tokens exchanged between + // the client and AI service during the defined period. + // + // +kubebuilder:validation:Minimum=1 + TotalTokenCount uint32 `json:"totalTokenCount,omitempty"` +} + +// AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy +type AIRateLimitPolicyStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// AIRateLimitPolicy is the Schema for the airatelimitpolicies API +type AIRateLimitPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AIRateLimitPolicySpec `json:"spec,omitempty"` + Status AIRateLimitPolicyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// AIRateLimitPolicyList contains a list of AIRateLimitPolicy +type AIRateLimitPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AIRateLimitPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AIRateLimitPolicy{}, &AIRateLimitPolicyList{}) +} \ No newline at end of file diff --git a/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go b/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go index 433662bac0..c048a5685a 100644 --- a/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go +++ b/common-go-libs/apis/dp/v1alpha3/zz_generated.deepcopy.go @@ -132,6 +132,131 @@ func (in *AIProviderStatus) DeepCopy() *AIProviderStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimit) DeepCopyInto(out *AIRateLimit) { + *out = *in + if in.TokenCount != nil { + in, out := &in.TokenCount, &out.TokenCount + *out = new(TokenCount) + **out = **in + } + if in.RequestCount != nil { + in, out := &in.RequestCount, &out.RequestCount + *out = new(RequestCount) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimit. +func (in *AIRateLimit) DeepCopy() *AIRateLimit { + if in == nil { + return nil + } + out := new(AIRateLimit) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicy) DeepCopyInto(out *AIRateLimitPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicy. +func (in *AIRateLimitPolicy) DeepCopy() *AIRateLimitPolicy { + if in == nil { + return nil + } + out := new(AIRateLimitPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AIRateLimitPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicyList) DeepCopyInto(out *AIRateLimitPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AIRateLimitPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicyList. +func (in *AIRateLimitPolicyList) DeepCopy() *AIRateLimitPolicyList { + if in == nil { + return nil + } + out := new(AIRateLimitPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AIRateLimitPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicySpec) DeepCopyInto(out *AIRateLimitPolicySpec) { + *out = *in + if in.Override != nil { + in, out := &in.Override, &out.Override + *out = new(AIRateLimit) + (*in).DeepCopyInto(*out) + } + if in.Default != nil { + in, out := &in.Default, &out.Default + *out = new(AIRateLimit) + (*in).DeepCopyInto(*out) + } + in.TargetRef.DeepCopyInto(&out.TargetRef) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicySpec. +func (in *AIRateLimitPolicySpec) DeepCopy() *AIRateLimitPolicySpec { + if in == nil { + return nil + } + out := new(AIRateLimitPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AIRateLimitPolicyStatus) DeepCopyInto(out *AIRateLimitPolicyStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIRateLimitPolicyStatus. +func (in *AIRateLimitPolicyStatus) DeepCopy() *AIRateLimitPolicyStatus { + if in == nil { + return nil + } + out := new(AIRateLimitPolicyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *APIPolicy) DeepCopyInto(out *APIPolicy) { *out = *in @@ -622,6 +747,21 @@ func (in *SubscriptionRateLimitPolicy) DeepCopy() *SubscriptionRateLimitPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TokenCount) DeepCopyInto(out *TokenCount) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TokenCount. +func (in *TokenCount) DeepCopy() *TokenCount { + if in == nil { + return nil + } + out := new(TokenCount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ValueDetails) DeepCopyInto(out *ValueDetails) { *out = *in diff --git a/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml b/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b30f687699 --- /dev/null +++ b/common-go-libs/config/crd/bases/dp.wso2.com_airatelimitpolicies.yaml @@ -0,0 +1,185 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: airatelimitpolicies.dp.wso2.com +spec: + group: dp.wso2.com + names: + kind: AIRateLimitPolicy + listKind: AIRateLimitPolicyList + plural: airatelimitpolicies + singular: airatelimitpolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AIRateLimitPolicy is the Schema for the airatelimitpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy + properties: + default: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the rule for request count quota. + properties: + requestsPerUnit: + format: int32 + type: integer + unit: + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + override: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the rule for request count quota. + properties: + requestsPerUnit: + format: int32 + type: integer + unit: + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + targetRef: + description: PolicyTargetReference identifies an API object to apply + a direct or inherited policy to. This should be used as part of + Policy resources that can target Gateway API resources. For more + information on how this policy attachment model works, and a sample + Policy resource, refer to the policy attachment documentation for + Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace is the namespace of the referent. When + unspecified, the local namespace is inferred. Even when policy + targets a resource in a different namespace, it MUST only apply + to traffic originating from the same namespace as the policy. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + type: object + status: + description: AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/common-go-libs/config/crd/kustomization.yaml b/common-go-libs/config/crd/kustomization.yaml index 10ba50c52e..5c3c767fc1 100644 --- a/common-go-libs/config/crd/kustomization.yaml +++ b/common-go-libs/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/dp.wso2.com_aiproviders.yaml - bases/cp.wso2.com_subscriptions.yaml +- bases/dp.wso2.com_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,12 +12,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_aiproviders.yaml #- patches/webhook_in_subscriptions.yaml +#- patches/webhook_in_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_aiproviders.yaml #- patches/cainjection_in_subscriptions.yaml +#- patches/cainjection_in_airatelimitpolicies.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml b/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..2d28892f72 --- /dev/null +++ b/common-go-libs/config/crd/patches/cainjection_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: airatelimitpolicies.dp.wso2.com diff --git a/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml b/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml new file mode 100644 index 0000000000..b2cde3fe7d --- /dev/null +++ b/common-go-libs/config/crd/patches/webhook_in_dp_airatelimitpolicies.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: airatelimitpolicies.dp.wso2.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml b/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml new file mode 100644 index 0000000000..c6afbc67c6 --- /dev/null +++ b/common-go-libs/config/rbac/dp_airatelimitpolicy_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-editor-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml b/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml new file mode 100644 index 0000000000..5fe04d1924 --- /dev/null +++ b/common-go-libs/config/rbac/dp_airatelimitpolicy_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view airatelimitpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: airatelimitpolicy-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: operator + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + name: airatelimitpolicy-viewer-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - get + - list + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get diff --git a/common-go-libs/config/rbac/role.yaml b/common-go-libs/config/rbac/role.yaml new file mode 100644 index 0000000000..2d74cacc5b --- /dev/null +++ b/common-go-libs/config/rbac/role.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/finalizers + verbs: + - update +- apiGroups: + - dp.wso2.com + resources: + - airatelimitpolicies/status + verbs: + - get + - patch + - update diff --git a/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml b/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml new file mode 100644 index 0000000000..4e1fcfd751 --- /dev/null +++ b/common-go-libs/config/samples/dp_v1alpha3_airatelimitpolicy.yaml @@ -0,0 +1,12 @@ +apiVersion: dp.wso2.com/v1alpha3 +kind: AIRateLimitPolicy +metadata: + labels: + app.kubernetes.io/name: airatelimitpolicy + app.kubernetes.io/instance: airatelimitpolicy-sample + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: operator + name: airatelimitpolicy-sample +spec: + # TODO(user): Add fields here diff --git a/common-go-libs/constants/constant.go b/common-go-libs/constants/constant.go index 14f01c9413..5450b8c852 100644 --- a/common-go-libs/constants/constant.go +++ b/common-go-libs/constants/constant.go @@ -20,6 +20,7 @@ package constants // Controller related constants const ( RatelimitController string = "RatelimitController" + AIRatelimitController string = "AIRatelimitController" ApplicationController string = "ApplicationController" SubscriptionController string = "SubscriptionController" ApplicationMappingController string = "ApplicationMappingController" diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java index c10358f336..238f744918 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/ConfigHolder.java @@ -308,6 +308,8 @@ private void loadTrustStore() { private void loadTrustedCertsToTrustStore() throws IOException { String truststoreFilePath = getEnvVarConfig().getTrustedAdapterCertsPath(); + String certificatePath = "/home/wso2/security/truststore/ratelimiter.crt"; + TLSUtils.addCertsToTruststore(trustStore, certificatePath); TLSUtils.addCertsToTruststore(trustStore, truststoreFilePath); } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java index a3b31bce7a..43dcade2a4 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/config/EnvVarConfig.java @@ -34,6 +34,8 @@ public class EnvVarConfig { private static final String OPA_CLIENT_PRIVATE_KEY_PATH = "OPA_CLIENT_PRIVATE_KEY_PATH"; private static final String OPA_CLIENT_PUBLIC_CERT_PATH = "OPA_CLIENT_PUBLIC_CERT_PATH"; private static final String ADAPTER_HOST = "ADAPTER_HOST"; + private static final String RATELIMITER_HOST = "RATELIMITER_HOST"; + private static final String RATELIMITER_PORT = "RATELIMITER_PORT"; private static final String ADAPTER_XDS_PORT = "ADAPTER_XDS_PORT"; private static final String COMMON_CONTROLLER_HOST = "COMMON_CONTROLLER_HOST"; private static final String COMMON_CONTROLLER_XDS_PORT = "COMMON_CONTROLLER_XDS_PORT"; @@ -68,6 +70,8 @@ public class EnvVarConfig { private static final String DEFAULT_ENFORCER_PUBLIC_CERT_PATH = "/home/wso2/security/keystore/mg.pem"; private static final String DEFAULT_ENFORCER_REGION_ID = "UNKNOWN"; private static final String DEFAULT_ADAPTER_HOST = "adapter"; + private static final String DEFAULT_RATELIMITER_HOST = "apk-test-wso2-apk-ratelimiter-service.apk.svc"; + private static final String DEFAULT_RATELIMITER_PORT = "8091"; private static final String DEFAULT_ADAPTER_XDS_PORT = "18000"; private static final String DEFAULT_COMMON_CONTROLLER_HOST = "common-controller"; private static final String DEFAULT_COMMON_CONTROLLER_XDS_PORT = "18002"; @@ -100,6 +104,8 @@ public class EnvVarConfig { private final String opaClientPrivateKeyPath; private final String opaClientPublicKeyPath; private final String adapterHost; + private final String ratelimiterHost; + private final String ratelimiterPort; private final String commonControllerHost; private final String enforcerLabel; private final String adapterXdsPort; @@ -144,6 +150,8 @@ private EnvVarConfig() { DEFAULT_ENFORCER_PUBLIC_CERT_PATH); enforcerLabel = retrieveEnvVarOrDefault(ENFORCER_LABEL, DEFAULT_ENFORCER_LABEL); adapterHost = retrieveEnvVarOrDefault(ADAPTER_HOST, DEFAULT_ADAPTER_HOST); + ratelimiterHost = retrieveEnvVarOrDefault(RATELIMITER_HOST, DEFAULT_RATELIMITER_HOST); + ratelimiterPort = retrieveEnvVarOrDefault(RATELIMITER_PORT, DEFAULT_RATELIMITER_PORT); adapterHostname = retrieveEnvVarOrDefault(ADAPTER_HOST_NAME, DEFAULT_ADAPTER_HOST_NAME); adapterXdsPort = retrieveEnvVarOrDefault(ADAPTER_XDS_PORT, DEFAULT_ADAPTER_XDS_PORT); commonControllerHost = retrieveEnvVarOrDefault(COMMON_CONTROLLER_HOST, DEFAULT_COMMON_CONTROLLER_HOST); @@ -240,6 +248,19 @@ public String getAdapterHost() { return adapterHost; } + public String getRatelimiterHost() { + return ratelimiterHost; + } + + public int getRatelimiterPort() { + try { + int port = Integer.parseInt(ratelimiterPort); + return port; + } catch (NumberFormatException e) { + return Integer.parseInt(DEFAULT_RATELIMITER_PORT); + } + } + public String getCommonControllerHost() { return commonControllerHost; } diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java new file mode 100644 index 0000000000..bda06b8e84 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/ExternalProcessorService.java @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2020, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.wso2.apk.enforcer.grpc; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.Value; +import io.envoyproxy.envoy.service.ext_proc.v3.BodyMutation; +import io.envoyproxy.envoy.service.ext_proc.v3.BodyResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.CommonResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.ExternalProcessorGrpc; +import io.envoyproxy.envoy.service.ext_proc.v3.HeadersResponse; +import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingRequest; +import io.envoyproxy.envoy.service.ext_proc.v3.ProcessingResponse; +import io.grpc.stub.StreamObserver; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.wso2.apk.enforcer.grpc.client.RatelimitClient; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This is the gRPC server written to match with the envoy ext-authz filter proto file. Envoy proxy call this service. + * This is the entry point to the filter chain process for a request. + */ +public class ExternalProcessorService extends ExternalProcessorGrpc.ExternalProcessorImplBase { + private static final Logger logger = LogManager.getLogger(ExternalProcessorService.class); + private static final String DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT = "airequesttokencount"; + private static final String DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT = "airesponsetokencount"; + private static final String DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT = "aitotaltokencount"; + RatelimitClient ratelimitClient = new RatelimitClient(); + @Override + public StreamObserver process( + final StreamObserver responseObserver) { + FilterMetadata filterMetadata = new FilterMetadata(); + System.out.println("process ...."); + return new StreamObserver() { + + @Override + public void onNext(ProcessingRequest request) { + System.out.println("on next ...."); + ProcessingRequest.RequestCase r = request.getRequestCase(); + System.out.println("case: " + r.name()); + switch (r) { + case REQUEST_HEADERS: + if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ + Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); + FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); + System.out.println("Metadata generated: "+ metadata); + filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; + filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; + filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; + } + responseObserver.onNext(ProcessingResponse.newBuilder().build()); + case RESPONSE_BODY: + if (!request.getAttributesMap().isEmpty() && request.getAttributesMap().get("envoy.filters.http.ext_proc") != null && request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata") != null){ + Value value = request.getAttributesMap().get("envoy.filters.http.ext_proc").getFieldsMap().get("xds.route_metadata"); + FilterMetadata metadata = convertStringToFilterMetadata(value.getStringValue()); + System.out.println("Metadata generated: "+ metadata); + filterMetadata.backendBasedAIRatelimitDescriptorValue = metadata.backendBasedAIRatelimitDescriptorValue; + filterMetadata.enableBackendBasedAIRatelimit = metadata.enableBackendBasedAIRatelimit; + filterMetadata.enableSubscriptionBasedAIRatelimit = metadata.enableSubscriptionBasedAIRatelimit; + } + System.out.println("In the response flow metadata descirtor:" + filterMetadata.backendBasedAIRatelimitDescriptorValue); + if (request.hasResponseBody()) { + String body = request.getResponseBody().getBody().toStringUtf8(); +// System.out.println("Body: " + body); + Usage usage = extractUsageFromBody(body, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); + if (usage == null) { + logger.error("Usage details not found.."); + System.out.println("Usage details not found.."); + responseObserver.onCompleted(); + return; + } + System.out.println("body: " +request.getResponseBody().getBody().toStringUtf8()); + if (filterMetadata.enableBackendBasedAIRatelimit) { + List configs = new ArrayList<>(); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_REQUEST_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getPrompt_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_RESPONSE_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getCompletion_tokens())); + configs.add(new RatelimitClient.KeyValueHitsAddend(DESCRIPTOR_KEY_FOR_AI_TOTAL_TOKEN_COUNT, filterMetadata.backendBasedAIRatelimitDescriptorValue, usage.getTotal_tokens())); + ratelimitClient.shouldRatelimit(configs); + } + responseObserver.onCompleted(); + } else { + System.out.println("Request does not have response body"); + responseObserver.onCompleted(); + } + + } + } + + @Override + public void onError(Throwable err) { + System.out.println("on error ...."+ err.getLocalizedMessage() + " " + err.getMessage() + " " + err.toString()+ " ****"); + } + + @Override + public void onCompleted() { + System.out.println("on completed ...."); + responseObserver.onCompleted(); + } + }; + } + + protected BodyResponse prepareBodyResponse() { + return BodyResponse.newBuilder() + .setResponse( + CommonResponse.newBuilder() + .setStatus(CommonResponse.ResponseStatus.CONTINUE) + .setBodyMutation(BodyMutation.newBuilder().build()) + .build()) + .build(); + } + + protected HeadersResponse prepareHeadersResponse() { + return HeadersResponse.newBuilder() + .setResponse( + CommonResponse.newBuilder() + .setStatus(CommonResponse.ResponseStatus.CONTINUE) + .setBodyMutation(BodyMutation.newBuilder().build()) + .build()) + .build(); + } + + // The FilterMetadata class as per your request + private static class FilterMetadata { + boolean enableSubscriptionBasedAIRatelimit; + boolean enableBackendBasedAIRatelimit; + String backendBasedAIRatelimitDescriptorValue; + @Override + public String toString() { + return "FilterMetadata{" + + "enableSubscriptionBasedAIRatelimit=" + enableSubscriptionBasedAIRatelimit + + ", enableBackendBasedAIRatelimit=" + enableBackendBasedAIRatelimit + + ", backendBasedAIRatelimitDescriptorValue='" + backendBasedAIRatelimitDescriptorValue + '\'' + + '}'; + } + } + + // Method to parse the string and create FilterMetadata object + public static FilterMetadata convertStringToFilterMetadata(String input) { + FilterMetadata metadata = new FilterMetadata(); + + // Regex patterns to extract specific fields + String backendValuePattern = "key: \"BackendBasedAIRatelimitDescriptorValue\".*?string_value: \"(.*?)\""; + String enableBackendPattern = "key: \"EnableBackendBasedAIRatelimit\".*?string_value: \"(.*?)\""; + String enableSubscriptionPattern = "key: \"EnableSubscriptionBasedAIRatelimit\".*?string_value: \"(.*?)\""; + + // Extract and assign to the FilterMetadata object + metadata.backendBasedAIRatelimitDescriptorValue = extractValue(input, backendValuePattern); + metadata.enableBackendBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableBackendPattern)); + metadata.enableSubscriptionBasedAIRatelimit = Boolean.parseBoolean(extractValue(input, enableSubscriptionPattern)); + + return metadata; + } + + // Helper method to extract value based on a regex pattern + private static String extractValue(String input, String pattern) { + Pattern p = Pattern.compile(pattern); + Matcher m = p.matcher(input); + if (m.find()) { + return m.group(1); + } + return null; + } + + + public static void main(String[] args) { + String input = "filter_metadata { key: \"envoy.filters.http.ext_proc\" value { fields { key: \"BackendBasedAIRatelimitDescriptorValue\" value { string_value: \"default-apk-backend-ratelimit-xxx-apk-backend-93fa16a00dbec9ff438aa40e6358d91dcdc22f48-api\" } } fields { key: \"EnableBackendBasedAIRatelimit\" value { string_value: \"true\" } } fields { key: \"EnableSubscriptionBasedAIRatelimit\" value { string_value: \"false\" } } } }"; + input = "{ \"choices\":[ { \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } }, \"finish_reason\":\"stop\", \"index\":0, \"logprobs\":null, \"message\":{ \"content\":\"Arr, matey! Ye be askin' a great question. Care for a parrot be a task that requires careful attention and love. Here be some tips to keep yer feathered friend happy and healthy: 1. Provide a proper cage: A parrot needs an adequate-sized cage to freely stretch its wings. Make sure it has enough space for perching, playing, and spreading those beautiful feathers. Also, ensure the bars are close enough together to prevent escape. 2. Nourishing grub: A parrot's diet be important. Offer a balanced diet of high-quality parrot pellets, fresh fruits, vegetables, and some seeds. Avoid avacados, chocolate, caffeine, and anythin' toxic to a bird's delicate system. 3. Fresh water: Change the water in yer parrot's bowl daily, matey. Keeps it clean and fresh. Parrots love to dunk their beaks, so ensure they have ample water for sippin' and splish-splashin'. 4. Feathered entertainment: Parrots be social creatures and need mental stimulation. Provide 'em with plenty of toys, such as ropes, bells, and puzzle toys, to keep 'em entertained. Rotate the toys frequently to avoid boredom. 5. Avast, matey! Give 'em attention: Parrots be fond of interaction with their human companions. Spend time talkin' to 'em, singin' shanties, and makin' 'em feel loved. They may even learn a few words or phrases! 6. Exercise be important: Encourage yer parrot to exercise its wings, me hearty. Free-flyin' in a safe, enclosed area be ideal, or let 'em out of their cage for supervised playtime. 7. Regular vet visits: Aye, take yer parrot to a qualified avian vet for regular check-ups. They'll make sure yer feathered friend's health be in shipshape and suggest any necessary vaccinations or treatments. 8. Aye, watch for signs of illness: Keep a keen eye on yer parrot for any signs of illness, such as changes in appetite, behavior, or feather condition. If ye spot any concerns, consult an avian vet right quick. Remember, matey, each parrot be unique, so get to know yer bird and pay attention to its specific needs. Aye, with proper care, ye and yer parrot will forge a bond that be stronger than the mightiest of pirate ships. Fair winds and happy parrot keepin'!\", \"role\":\"assistant\" } } ], \"created\":1724232516, \"id\":\"chatcmpl-9ybvo8dQte9Hb0IkD2NCaOME9Q1LH\", \"model\":\"gpt-35-turbo\", \"object\":\"chat.completion\", \"prompt_filter_results\":[ { \"prompt_index\":0, \"content_filter_results\":{ \"hate\":{ \"filtered\":false, \"severity\":\"safe\" }, \"self_harm\":{ \"filtered\":false, \"severity\":\"safe\" }, \"sexual\":{ \"filtered\":false, \"severity\":\"safe\" }, \"violence\":{ \"filtered\":false, \"severity\":\"safe\" } } } ], \"system_fingerprint\":null, \"usage\":{ \"completion_tokens\":514, \"prompt_tokens\":33, \"total_tokens\":547 } }"; + Usage usage = extractUsageFromBody(input, "usage.completion_tokens", "usage.prompt_tokens", "usage.total_tokens"); + System.out.println(usage.completion_tokens); +// FilterMetadata metadata = convertStringToFilterMetadata(input); +// System.out.println(metadata.backendBasedAIRatelimitDescriptorValue); // Printing the FilterMetadata object + } + + public static String sanitize(String input) { + // Replace all newline characters and tabs with a space + return input.replaceAll("[\\t\\n\\r]+", " ").trim(); + } + + private static Usage extractUsageFromBody(String body, String completionTokenPath, String promptTokenPath, String totalTokenPath) { + body = sanitize(body); + ObjectMapper mapper = new ObjectMapper(); + try { + Usage usage = new Usage(); + // Parse the JSON string + JsonNode rootNode = mapper.readTree(body); + // Extract prompt token count + String[] keysForPromtTokens = promptTokenPath.split("\\."); + JsonNode currentNodeForPromtToken = null; + if (rootNode.has(keysForPromtTokens[0])) { + currentNodeForPromtToken = rootNode.get(keysForPromtTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForPromtTokens.length; i++) { + if (currentNodeForPromtToken.has(keysForPromtTokens[i])) { + currentNodeForPromtToken = currentNodeForPromtToken.get(keysForPromtTokens[i]); + } else { + return null; + } + } + usage.setPrompt_tokens(currentNodeForPromtToken.asInt()); + + // Extract completion token count + String[] keysForCompletionTokens = completionTokenPath.split("\\."); + JsonNode currentNodeForCompletionToken = null; + if (rootNode.has(keysForCompletionTokens[0])) { + currentNodeForCompletionToken = rootNode.get(keysForCompletionTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForCompletionTokens.length; i++) { + if (currentNodeForCompletionToken.has(keysForCompletionTokens[i])) { + currentNodeForCompletionToken = currentNodeForCompletionToken.get(keysForCompletionTokens[i]); + } else { + return null; + } + } + usage.setCompletion_tokens(currentNodeForCompletionToken.asInt()); + + // Extract total token count + String[] keysForTotalTokens = totalTokenPath.split("\\."); + JsonNode currentNodeForTotalToken = null; + if (rootNode.has(keysForTotalTokens[0])) { + currentNodeForTotalToken = rootNode.get(keysForTotalTokens[0]); + } else { + return null; + } + for (int i = 1; i < keysForTotalTokens.length; i++) { + if (currentNodeForTotalToken.has(keysForTotalTokens[i])) { + currentNodeForTotalToken = currentNodeForTotalToken.get(keysForTotalTokens[i]); + } else { + return null; + } + } + usage.setTotal_tokens(currentNodeForTotalToken.asInt()); + System.out.println("Usage extracted: "+ usage); + return usage; + + } catch (JsonProcessingException e) { + logger.error(String.format("Unexpected error while extracting usage from the body: %s", body), e); + return null; + } + } + + public static class Usage { + private int completion_tokens; + private int prompt_tokens; + private int total_tokens; + + // Getters and Setters + public int getCompletion_tokens() { + return completion_tokens; + } + + public void setCompletion_tokens(int completion_tokens) { + this.completion_tokens = completion_tokens; + } + + public int getPrompt_tokens() { + return prompt_tokens; + } + + public void setPrompt_tokens(int prompt_tokens) { + this.prompt_tokens = prompt_tokens; + } + + public int getTotal_tokens() { + return total_tokens; + } + + public void setTotal_tokens(int total_tokens) { + this.total_tokens = total_tokens; + } + + @Override + public String toString() { + return String.format("%s_%s_%s", prompt_tokens, completion_tokens, total_tokens); + } + } + +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java new file mode 100644 index 0000000000..3034296060 --- /dev/null +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/grpc/client/RatelimitClient.java @@ -0,0 +1,97 @@ +package org.wso2.apk.enforcer.grpc.client; + +import io.envoyproxy.envoy.extensions.common.ratelimit.v3.RateLimitDescriptor; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitRequest; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitServiceGrpc; +import io.envoyproxy.envoy.service.ratelimit.v3.RateLimitResponse; +import io.grpc.ManagedChannel; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import org.wso2.apk.enforcer.config.ConfigHolder; + +import java.io.File; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.net.ssl.SSLException; + +public class RatelimitClient { + RateLimitServiceGrpc.RateLimitServiceBlockingStub stub; + private final ExecutorService executorService; // Add an ExecutorService field + + public RatelimitClient(){//String server, int port) { + System.out.println("Ratelimitclient construct"); + File certFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPublicKeyPath()).toFile(); + File keyFile = Paths.get(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerPrivateKeyPath()).toFile(); + SslContext sslContext = null; + try { + sslContext = GrpcSslContexts + .forClient() + .trustManager(ConfigHolder.getInstance().getTrustManagerFactory()) + .keyManager(certFile, keyFile) + .build(); + } catch (SSLException e) { + System.out.println("Error while generating SSL Context."+ e); + } + String rlHost = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterHost(); + int port = ConfigHolder.getInstance().getEnvVarConfig().getRatelimiterPort(); + System.out.println("rl host and port "+ rlHost + port); + ManagedChannel channel = NettyChannelBuilder.forAddress(rlHost, port) + .useTransportSecurity() + .sslContext(sslContext) + .build(); + this.stub = RateLimitServiceGrpc.newBlockingStub(channel); + // Initialize the ExecutorService + this.executorService = Executors.newFixedThreadPool(10); + } + + public void shouldRatelimit(List configs) { +// System.out.println("RL task submitted"); + executorService.submit(() -> { + System.out.println("Ratelimitclient test"); + for (KeyValueHitsAddend config : configs) { + System.out.println("For: " + config.getKey()); + RateLimitDescriptor descriptor = RateLimitDescriptor.newBuilder() + .addEntries(RateLimitDescriptor.Entry.newBuilder().setKey(config.getKey()).setValue(config.getValue()).build()) + .build(); + RateLimitRequest rateLimitRequest = RateLimitRequest.newBuilder() + .addDescriptors(descriptor) + .setDomain("Default") + .setHitsAddend(config.getHitsAddend()) + .build(); + RateLimitResponse rateLimitResponse = stub.shouldRateLimit(rateLimitRequest); + System.out.println(rateLimitResponse.getOverallCode()); + } + }); + System.out.println("RL task submitted"); + + } + + public static class KeyValueHitsAddend { + private String key; + private String value; + private int hitsAddend; + + public KeyValueHitsAddend(String key, String value, int hitsAddend) { + this.key = key; + this.value = value; + this.hitsAddend = hitsAddend; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + public int getHitsAddend() { + return hitsAddend; + } + } + + +} diff --git a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java index 6f31ffc2de..d849dc648a 100644 --- a/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java +++ b/gateway/enforcer/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/server/AuthServer.java @@ -36,6 +36,7 @@ import org.wso2.apk.enforcer.config.dto.ThreadPoolConfig; import org.wso2.apk.enforcer.discovery.ConfigDiscoveryClient; import org.wso2.apk.enforcer.grpc.ExtAuthService; + import org.wso2.apk.enforcer.grpc.ExternalProcessorService; import org.wso2.apk.enforcer.grpc.HealthService; import org.wso2.apk.enforcer.grpc.interceptors.AccessLogInterceptor; import org.wso2.apk.enforcer.grpc.interceptors.OpenTelemetryInterceptor; @@ -158,12 +159,14 @@ private static Server initServer() throws SSLException { EnforcerWorkerPool enforcerWorkerPool = new EnforcerWorkerPool(threadPoolConfig.getCoreSize(), threadPoolConfig.getMaxSize(), threadPoolConfig.getKeepAliveTime(), threadPoolConfig.getQueueSize(), Constants.EXTERNAL_AUTHZ_THREAD_GROUP, Constants.EXTERNAL_AUTHZ_THREAD_ID); + System.out.println("test"); return NettyServerBuilder.forPort(authServerConfig.getPort()) .keepAliveTime(authServerConfig.getKeepAliveTime(), TimeUnit.SECONDS).bossEventLoopGroup(bossGroup) .workerEventLoopGroup(workerGroup) .addService(ServerInterceptors.intercept(new ExtAuthService(), new OpenTelemetryInterceptor(), new AccessLogInterceptor())) .addService(new HealthService()) + .addService(new ExternalProcessorService()) // .addService(ServerInterceptors.intercept(new WebSocketFrameService(), new AccessLogInterceptor())) .maxInboundMessageSize(authServerConfig.getMaxMessageSize()) .maxInboundMetadataSize(authServerConfig.getMaxHeaderLimit()).channelType(NioServerSocketChannel.class) diff --git a/gateway/router/Dockerfile b/gateway/router/Dockerfile index 325a988056..5d45304b53 100644 --- a/gateway/router/Dockerfile +++ b/gateway/router/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ----------------------------------------------------------------------- -FROM envoyproxy/envoy:v1.29.1 +FROM envoyproxy/envoy:v1.31.0 LABEL maintainer="WSO2 Docker Maintainers " RUN apt-get update && apt-get upgrade -y && apt-get install -y curl diff --git a/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml b/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml new file mode 100644 index 0000000000..8eb400bcdc --- /dev/null +++ b/helm-charts/crds/dp.wso2.com_airatelimitpolicies.yaml @@ -0,0 +1,201 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: airatelimitpolicies.dp.wso2.com +spec: + group: dp.wso2.com + names: + kind: AIRateLimitPolicy + listKind: AIRateLimitPolicyList + plural: airatelimitpolicies + singular: airatelimitpolicy + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AIRateLimitPolicy is the Schema for the airatelimitpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AIRateLimitPolicySpec defines the desired state of AIRateLimitPolicy + properties: + default: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the request based ratelimit + configuration + properties: + requestsPerUnit: + description: RequestPerUnit is the number of requests allowed + per unit time + format: int32 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + override: + description: AIRateLimit defines the AI ratelimit configuration + properties: + organization: + type: string + requestCount: + description: RequestCount defines the request based ratelimit + configuration + properties: + requestsPerUnit: + description: RequestPerUnit is the number of requests allowed + per unit time + format: int32 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + tokenCount: + description: TokenCount defines the Token based ratelimit configuration + properties: + requestTokenCount: + description: RequestTokenCount specifies the maximum number + of tokens allowed in AI requests within a given unit of + time. This value limits the token count sent by the client + to the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + responseTokenCount: + description: ResponseTokenCount specifies the maximum number + of tokens allowed in AI responses within a given unit of + time. This value limits the token count received by the + client from the AI service over the defined period. + format: int32 + minimum: 1 + type: integer + totalTokenCount: + description: TotalTokenCount represents the maximum allowable + total token count for both AI requests and responses within + a specified unit of time. This value sets the limit for + the number of tokens exchanged between the client and AI + service during the defined period. + format: int32 + minimum: 1 + type: integer + unit: + description: Unit is the unit of the requestsPerUnit + enum: + - Minute + - Hour + - Day + type: string + type: object + type: object + targetRef: + description: PolicyTargetReference identifies an API object to apply + a direct or inherited policy to. This should be used as part of + Policy resources that can target Gateway API resources. For more + information on how this policy attachment model works, and a sample + Policy resource, refer to the policy attachment documentation for + Gateway API. + properties: + group: + description: Group is the group of the target resource. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the target resource. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the target resource. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: Namespace is the namespace of the referent. When + unspecified, the local namespace is inferred. Even when policy + targets a resource in a different namespace, it MUST only apply + to traffic originating from the same namespace as the policy. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - group + - kind + - name + type: object + type: object + status: + description: AIRateLimitPolicyStatus defines the observed state of AIRateLimitPolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml index 3d6fc60622..bf6217a4ff 100644 --- a/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml +++ b/helm-charts/templates/data-plane/gateway-components/gateway-runtime/gateway-runtime-deployment.yaml @@ -70,6 +70,8 @@ spec: value: {{ template "apk-helm.resource.prefix" . }}-common-controller-service.{{ .Release.Namespace }}.svc - name: COMMON_CONTROLLER_HOST value: {{ template "apk-helm.resource.prefix" . }}-common-controller-service.{{ .Release.Namespace }}.svc + - name: RATELIMITER_HOST + value: {{ template "apk-helm.resource.prefix" . }}-ratelimiter-service.{{ .Release.Namespace }}.svc - name: ENFORCER_PRIVATE_KEY_PATH value: /home/wso2/security/keystore/enforcer.key - name: ENFORCER_PUBLIC_CERT_PATH @@ -80,6 +82,8 @@ spec: value: "/home/wso2/security/truststore" - name: ADAPTER_XDS_PORT value : "18000" + - name: RATELIMITER_PORT + value : "8091" - name: COMMON_CONTROLLER_XDS_PORT value : "18002" - name: COMMON_CONTROLLER_REST_PORT @@ -96,9 +100,9 @@ spec: value: admin - name: JAVA_OPTS {{- if and .Values.wso2.apk.metrics .Values.wso2.apk.metrics.enabled }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml + value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 -Dapk.jmx.metrics.enabled=true -javaagent:/home/wso2/lib/jmx_prometheus_javaagent-0.20.0.jar=18006:/tmp/metrics/prometheus-jmx-config-enforcer.yml {{- else }} - value: -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 + value: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006 -Dhttpclient.hostnameVerifier=AllowAll -Xms512m -Xmx512m -XX:MaxRAMFraction=2 {{- end }} {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers }} {{- $defaultPublisherSecretName := "" }} @@ -255,6 +259,22 @@ spec: mountPath: /home/wso2/security/truststore/idp-tls.pem subPath: {{ .Values.wso2.apk.idp.tls.fileName }} {{ end }} + {{ if and .Values.wso2.apk.dp.enabled .Values.wso2.apk.dp.ratelimiter.enabled }} + - name: ratelimiter-truststore-secret-volume + mountPath: /home/wso2/security/truststore/ratelimiter.crt + {{- if and .Values.wso2.apk.dp.ratelimiter.deployment.configs .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls }} + subPath: {{ .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls.certFilename | default "tls.crt" }} + {{- else }} + subPath: tls.crt + {{- end }} + - name: ratelimiter-truststore-secret-volume + mountPath: /home/wso2/security/truststore/ratelimiter-ca.crt + {{- if and .Values.wso2.apk.dp.ratelimiter.deployment.configs .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls }} + subPath: {{ .Values.wso2.apk.dp.ratelimiter.deployment.configs.tls.certCAFilename | default "ca.crt" }} + {{- else }} + subPath: ca.crt + {{- end }} + {{ end }} readinessProbe: exec: command: [ "sh", "check_health.sh" ] diff --git a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml index ccbb849a5d..2a153db946 100644 --- a/helm-charts/templates/serviceAccount/apk-cluster-role.yaml +++ b/helm-charts/templates/serviceAccount/apk-cluster-role.yaml @@ -92,6 +92,15 @@ rules: - apiGroups: [ "dp.wso2.com" ] resources: [ "ratelimitpolicies/status" ] verbs: [ "get","patch","update" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies" ] + verbs: [ "get","list","watch","update","delete","create" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies/finalizers" ] + verbs: [ "update" ] + - apiGroups: [ "dp.wso2.com" ] + resources: [ "airatelimitpolicies/status" ] + verbs: [ "get","patch","update" ] - apiGroups: [ "coordination.k8s.io" ] resources: [ "leases" ] verbs: [ "get","list","watch","update","patch","create","delete" ] diff --git a/libs.versions.toml b/libs.versions.toml index 100dd115ac..64f34aa1d2 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -131,12 +131,12 @@ commons-logging = "1.1.1" commons-pool = "1.5.6.wso2v1" commons-validator = "1.7" cxf = "3.5.4" -envoyproxy = "1.0.42" +envoyproxy = "1.0.46" fasterxml-woodstox="6.4.0" everit = "1.5.0.wso2.v2" geronimo = "1.1.1.wso2v1" graphql = "21.1" -grpc = "1.53.0" +grpc = "1.62.2" gson = "2.10" guava = "32.1.2-jre" hibernate-validator = "5.4.3.Final"