5.1.3 (#70)

* 20240917-01 dev Moesif integration

* 20240917-01
Added API visibility integration with Moesif
Postman collection updated

FEATURES.md

@@ -60,6 +60,10 @@ Declaration path `.declaration.http.servers[].locations[].apigateway` defines th
 - `developer_portal.type` - developer portal type. `redocly` and `backstage` are currently supported
 - `developer_portal.redocly.*` - Redocly-based developer portal parameters. See the [Postman collection](/contrib/postman)
 - `developer_portal.backstage.*` - Backstage-based developer portal parameters. See the [Postman collection](/contrib/postman)
+- `visibility[]` - API Gateway visibility
+  - `visibility[].enabled` - enable/disable API gateway visibility
+  - `visibility[].type` - visibility integration type. `moesif` is currently supported
+  - `visibility[].moesif.*` - Moesif visibility parameters. See the [Postman collection](/contrib/postman)
 - `authentication` - optional, used to enforce authentication at the API Gateway level
 - `authentication.client[]` - authentication profile names
 - `authentication.enforceOnPaths` - if set to `true` authentication is enforced on all API endpoints listed under `authentication.paths`. if set to `false` authentication is enforced on all API endpoints but those listed under `authentication.paths`
@@ -80,6 +84,13 @@ See the [Postman collection](/contrib/) for usage examples
 | Redocly       | X        | X         | X        | Developer portal published by NGINX Plus |
 | Backstage.io  |          | X         | X        | Backstage YAML manifest generated        |
 
+### NGINX API Gateway use case - Visibility
+
+| Type         | API v4.2 | API v5.0 | API v5.1 | Notes                                                                                         |
+|--------------|----------|----------|----------|-----------------------------------------------------------------------------------------------|
+| Moesif       |          |          | X        | Integration with Moesif - see https://www.moesif.com/docs/server-integration/nginx-openresty/ |
+
+
 ### Client authentication
 
 | Type | Description          | API v4.2 | API v5.0 | API v5.1                            | Notes                               |

README.md

@@ -54,6 +54,7 @@ stateDiagram-v2
     DEVP: Developer Portal Service
     OUTPUT: Output
     REDIS: Redis
+    3RDPARTY: 3rd Party integrations
 
     DevOps --> Pipeline
     Pipeline --> INPUT
@@ -62,6 +63,8 @@ stateDiagram-v2
     NDAPI --> OUTPUT
     NDAPI --> SOT
     SOT --> NDAPI
+    NDAPI --> 3RDPARTY
+    3RDPARTY --> NDAPI
     NDAPI --> REDIS
     REDIS --> NDAPI
     OUTPUT --> NIM

etc/config.toml

@@ -14,6 +14,7 @@ mimetypes = "nginx-conf/mime.types"
 
 httpconf = "http.tmpl"
 apigwconf = "apigateway.tmpl"
+visibility_root = "visibility"
 streamconf = "stream.tmpl"
 configmap = "configmap.tmpl"
 
@@ -54,6 +55,7 @@ nginx_conf = '/etc/nginx/nginx.conf'
 config_dir = '/etc/nginx'
 certs_dir = '/etc/nginx/ssl'
 apigw_dir = '/etc/nginx/apigateway'
+visibility_dir = '/etc/nginx/visibility'
 devportal_dir = '/etc/nginx/devportal'
 auth_client_dir = '/etc/nginx/authn/client'
 auth_server_dir = '/etc/nginx/authn/server'

src/V5_1_CreateConfig.py

@@ -444,6 +444,52 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
                                         "message": {"status_code": status, "message":
                                             {"code": status, "content": f"invalid server authentication profile [{authServerProfile['profile']}] in location [{loc['uri']}]"}}}
 
+                    # API Gateway visualization integrations
+                    apiGwVisibilityIntegrations = {}
+
+                    if loc['apigateway'] and loc['apigateway']['visibility']:
+                        visibility_integrations = loc['apigateway']['visibility']
+                        for i in range(len(visibility_integrations)):
+                            vis = visibility_integrations[i]
+
+                            if vis['enabled'] == True:
+                                apiGwVisibilityIntegrations[ vis['type'] ] = True
+
+                                if vis['type'].lower() == 'moesif':
+                                    # Moesif integration
+
+                                    # Add the rendered Moesif visibility configuration snippet as a config file in the staged configuration - HTTP context
+                                    templateName = NcgConfig.config['templates'][
+                                                       'visibility_root'] + "/moesif/http.tmpl"
+                                    renderedMoesifHTTP = j2_env.get_template(templateName).render(
+                                        vis=vis, loc=loc, ncgconfig=NcgConfig.config)
+
+                                    b64renderedMoesifHTTP = base64.b64encode(
+                                        bytes(renderedMoesifHTTP, 'utf-8')).decode(
+                                        'utf-8')
+                                    moesifHTTPConfigFile = {'contents': b64renderedMoesifHTTP,
+                                                            'name': NcgConfig.config['nms'][
+                                                                        'visibility_dir'] +
+                                                                    loc['uri'] + "-moesif-http.conf"}
+
+                                    auxFiles['files'].append(moesifHTTPConfigFile)
+
+                                    # Add the rendered Moesif visibility configuration snippet as a config file in the staged configuration - server context
+                                    templateName = NcgConfig.config['templates'][
+                                                       'visibility_root'] + "/moesif/server.tmpl"
+                                    renderedMoesifServer = j2_env.get_template(templateName).render(
+                                        vis=vis, loc=loc, ncgconfig=NcgConfig.config)
+
+                                    b64renderedMoesifServer = base64.b64encode(
+                                        bytes(renderedMoesifServer, 'utf-8')).decode(
+                                        'utf-8')
+                                    moesifServerConfigFile = {'contents': b64renderedMoesifServer,
+                                                              'name': NcgConfig.config['nms'][
+                                                                          'visibility_dir'] +
+                                                                      loc['uri'] + "-moesif-server.conf"}
+
+                                    auxFiles['files'].append(moesifServerConfigFile)
+
                     # API Gateway provisioning
                     if loc['apigateway'] and loc['apigateway']['api_gateway'] and loc['apigateway']['api_gateway']['enabled'] and loc['apigateway']['api_gateway']['enabled'] == True:
                         openApiAuthProfile = loc['apigateway']['openapi_schema']['authentication']
@@ -458,7 +504,7 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
                         # API Gateway configuration template rendering
                         if apiGatewayConfigDeclaration:
                             apiGatewaySnippet = j2_env.get_template(NcgConfig.config['templates']['apigwconf']).render(
-                                declaration=apiGatewayConfigDeclaration, ncgconfig=NcgConfig.config)
+                                declaration=apiGatewayConfigDeclaration, enabledVisibility=apiGwVisibilityIntegrations, ncgconfig=NcgConfig.config)
                             apiGatewaySnippetb64 = base64.b64encode(bytes(apiGatewaySnippet, 'utf-8')).decode('utf-8')
 
                             newAuxFile = {'contents': apiGatewaySnippetb64, 'name': NcgConfig.config['nms']['apigw_dir'] +
@@ -485,14 +531,14 @@ def createconfig(declaration: ConfigDeclaration, apiversion: str, runfromautosyn
                                                                                loc['apigateway']['developer_portal']['redocly']['uri']}
                             auxFiles['files'].append(newAuxFile)
 
+                            ### / Redocly developer portal - Add optional API Developer portal HTML files
                         elif loc['apigateway']['developer_portal']['type'].lower() == 'backstage':
                             ### Backstage developer portal - Create Kubernetes Backstage manifest
                             backstageManifest = j2_env.get_template(f"{NcgConfig.config['templates']['devportal_root']}/backstage.tmpl").render(
                                 declaration=loc['apigateway']['developer_portal']['backstage'], openAPISchema = v5_1.MiscUtils.json_to_yaml(openAPISchemaJSON), ncgconfig=NcgConfig.config)
 
                             extraOutputManifests.append(backstageManifest)
-
-                        ### / Backstage developer portal - Create Kubernetes Backstage manifest
+                            ### / Backstage developer portal - Create Kubernetes Backstage manifest
 
                     # Check rate limit profile name validity
                     if loc['rate_limit'] is not None:

src/V5_1_NginxConfigDeclaration.py

@@ -831,10 +831,40 @@ def check_type(self) -> 'DeveloperPortal':
         return self
 
 
+class Visibility_Moesif(BaseModel, extra="forbid"):
+    application_id: str = ""
+    plugin_path: Optional[str] = "/usr/local/share/lua/5.1/resty/moesif"
+
+
+class Visibility(BaseModel, extra="forbid"):
+    enabled: Optional[bool] = False
+    type: str = ""
+    moesif: Optional[Visibility_Moesif] = {}
+
+    @model_validator(mode='after')
+    def check_type(self) -> 'Visibility':
+        _enabled, _type, _moesif = self.enabled, self.type, self.moesif
+
+        valid = ['moesif']
+
+        if _enabled == True and _type not in valid:
+            raise ValueError(f"Invalid visibility type [{_type}] must be one of {str(valid)}")
+
+        isError = False
+
+        if _type == 'moesif' and not _moesif:
+            isError = True
+
+        if isError:
+            raise ValueError(f"Missing visibility data for type [{_type}]")
+
+        return self
+
 class APIGateway(BaseModel, extra="forbid"):
     openapi_schema: Optional[ObjectFromSourceOfTruth] = {}
     api_gateway: Optional[API_Gateway] =  {}
     developer_portal: Optional[DeveloperPortal] = {}
+    visibility: Optional[List[Visibility]] = []
     rate_limit: Optional[List[RateLimitApiGw]] = []
     authentication: Optional[APIGatewayAuthentication] = {}
     authorization: Optional[List[APIGatewayAuthorization]] = []

src/v5_1/NGINXOneOutput.py

@@ -38,7 +38,7 @@ def NGINXOneOutput(d, declaration: ConfigDeclaration, apiversion: str, b64HttpCo
               b64StreamConf: str,configFiles = {}, auxFiles = {},
               runfromautosync: bool = False,
               configUid: str = ""):
-    # NGINX Instance Manager Staged Configuration publish
+    # NGINX One Cloud Console Staged Configuration publish
 
     nOneToken = v5_1.MiscUtils.getDictKey(d, 'output.nginxone.token')
     nOneConfigSyncGroup = v5_1.MiscUtils.getDictKey(d, 'output.nginxone.configsyncgroup')
@@ -213,7 +213,7 @@ def NGINXOneOutput(d, declaration: ConfigDeclaration, apiversion: str, b64HttpCo
         #
         #### / NGINX App Protect policies support
 
-        ### Publish staged config to instance group
+        ### Publish staged config to config sync group
         r = requests.put(url=f'{nOneUrl}/api/nginx/one/namespaces/{nOneNamespace}/config-sync-groups/{igUid}/config',
                           data=json.dumps(stagedConfig),
                           headers={'Content-Type': 'application/json', "Authorization": f"Bearer APIToken {nOneToken}"},

templates/v5.1/apigateway.tmpl

@@ -18,6 +18,10 @@
 # Strip base URI: {{ declaration.location.apigateway.api_gateway.strip_uri }}
 # Destination server: {{ destination_server }}
 
+{% for v in enabledVisibility %}
+include "{{ ncgconfig.nms.visibility_dir }}{{ declaration.location.uri }}-{{ v }}-server.conf";
+{% endfor %}
+
 {% if declaration.paths -%}
 {% for path in declaration.paths %}
 location {% if '{' not in path.path %}={% else %}~{% endif %} {{ declaration.location.uri }}{{ path.path | regex_replace('{(.*?)}','(.*)') }} {

templates/v5.1/http.tmpl

@@ -49,6 +49,7 @@ include "{{ ncgconfig.nms.authz_client_dir }}/{{ authzprofile.name | replace(" "
 
 
 {# --- Upstreams section --- #}
+# Upstreams
 {% if declaration.upstreams %}
 {% for u in declaration.upstreams %}
 {% if u.name %}
@@ -58,13 +59,17 @@ include "{{ ncgconfig.nms.upstream_http_dir }}/{{ u.name | replace(' ', '_') }}.
 {% endif %}
 
 {# --- Rate limit section --- #}
-
+# Rate limiting zones
 {% if declaration.rate_limit %}
 {% for rl in declaration.rate_limit %}
 limit_req_zone {{ rl.key }} zone={{ rl.name }}:{{ rl.size }} rate={{ rl.rate }};
 {% endfor %}
 {% endif %}
 
+{# --- Visibility integration section --- #}
+# Visibility integrations
+include "{{ ncgconfig.nms.visibility_dir }}/*-http.conf";
+
 {# --- Server section for NGINX Plus API --- #}
 
 {% if declaration.nginx_plus_api %}

templates/v5.1/visibility/moesif/http.tmpl

@@ -0,0 +1,14 @@
+# Moesif integration - https://www.moesif.com/docs/server-integration/nginx-openresty/
+# URI: {{ loc.uri }}
+# application ID: {{ vis.moesif.application_id }}
+
+lua_shared_dict moesif_conf 5m;
+
+init_by_lua_block {
+   local config = ngx.shared.moesif_conf;
+   config:set("application_id", "{{ vis.moesif.application_id }}")
+}
+
+lua_package_cpath ";;${prefix}?.so;${prefix}src/?.so;/usr/share/lua/5.1/lua/resty/moesif/?.so;/usr/share/lua/5.1/?.so;/usr/lib64/lua/5.1/?.so;/usr/lib/lua/5.1/?.so;/usr/local/openresty/luajit/share/lua/5.1/lua/resty?.so;/usr/local/share/lua/5.1/resty/moesif/?.so;{{ vis.moesif.plugin_path }}/?.so";
+lua_package_path ";;${prefix}?.lua;${prefix}src/?.lua;/usr/share/lua/5.1/lua/resty/moesif/?.lua;/usr/share/lua/5.1/?.lua;/usr/lib64/lua/5.1/?.lua;/usr/lib/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/lua/resty?.lua;/usr/local/share/lua/5.1/resty/moesif/?.lua;{{ vis.moesif.plugin_path }}/?.lua";
+

templates/v5.1/visibility/moesif/server.tmpl

@@ -0,0 +1,21 @@
+# Moesif integration - https://www.moesif.com/docs/server-integration/nginx-openresty/
+# URI: {{ loc.uri }}
+# application ID: {{ vis.moesif.application_id }}
+
+# Define the variables Moesif requires
+set $moesif_user_id nil;
+set $moesif_company_id nil;
+set $moesif_req_body nil;
+set $moesif_res_body nil;
+
+# Optionally, set moesif_user_id and moesif_company_id such from a request header or NGINX var to identify customer
+header_filter_by_lua_block  {
+  ngx.var.moesif_user_id = ngx.req.get_headers()["X-User-Id"]
+  ngx.var.moesif_company_id = ngx.req.get_headers()["X-Company-Id"]
+}
+
+# Add Moesif plugin
+access_by_lua_file {{ vis.moesif.plugin_path }}/read_req_body.lua;
+body_filter_by_lua_file {{ vis.moesif.plugin_path }}/read_res_body.lua;
+log_by_lua_file {{ vis.moesif.plugin_path }}/send_event.lua;
+