Add LogEvents interceptor, update test schemas

coldbox-modules · Sep 25, 2024 · f9fc2a1 · f9fc2a1
1 parent 51552fb
commit f9fc2a1
Show file tree

Hide file tree

Showing 19 changed files with 297 additions and 62 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -64,7 +64,6 @@ jobs:
         env:
           ENVIRONMENT: development
         run: |
-          cd test-harness
           box server start serverConfigFile="server-${{ matrix.cfengine }}.json" --noSaveSettings --debug
           cd ../
           curl http://127.0.0.1:60299

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 .vscode
 
+.engine/**
 .artifacts/**
 .tmp/**
 

diff --git a/.markdownlint.json b/.markdownlint.json
@@ -8,7 +8,7 @@
 	"no-multiple-blanks": {
 		"maximum": 2
 	},
-	"no-duplicate-header" : {
-		"siblings_only" : true
-	}
+	"no-duplicate-header" : false,
+	"no-duplicate-heading" : false,
+	"no-inline-html" : false
 }
diff --git a/ModuleConfig.cfc b/ModuleConfig.cfc
@@ -33,7 +33,7 @@ component {
 			"applicationName" 		: getSystemSetting( "LOGSTASH_APPLICATION_NAME", server.coldfusion.productname eq "Lucee" ? getApplicationSettings().name : getApplicationMetadata().name ),
 			// Whether to enable the API endpoints to receive log messages
 			"enableAPI" 			: getSystemSetting( "LOGSTASH_ENABLE_API", true ),
-			// Whether to automatically enabled log appenders
+			// Whether to automatically enable the logstash logging appender
 			"enableAppenders" 		: getSystemSetting( "LOGSTASH_ENABLE_APPENDERS", true ),
 			// The type of transmission mode for this module - `direct` or `api`
 			"transmission" 			: getSystemSetting( "LOGSTASH_TRANSMISSION_METHOD", "direct" ),
@@ -68,7 +68,9 @@ component {
 			"indexPrefix"           : getSystemSetting( "LOGSTASH_INDEX_PREFIX", "" ),
 			"migrateIndices"  		: getSystemSetting( "LOGSTASH_MIGRATE_V2", false ),
 			// Whether to throw an error when a log document fails to save
-			"throwOnError"    		: true
+			"throwOnError"    		: true,
+			// An array of detached appenders which can be used with the `writeToAppender` interception point or directly through the elasticsearch module AppenderService
+			"detachedAppenders" 	: []
         };
 
         // Try to look up the release based on a box.json
@@ -92,27 +94,15 @@ component {
 
         interceptors = [
             //API Security Interceptor
-            { class="logstash.interceptors.APISecurity" }
+            { class="logstash.interceptors.APISecurity" },
+            { class="logstash.interceptors.LogEvents" }
 		];
 
-		if( settings.enableAPI ){
-			routes = [
-				// Module Entry Point
-				{
-					pattern = "/api/logs",
-					handler = "API",
-					action = {
-						"HEAD"		: "onInvalidHTTPMethod",
-						"OPTIONS"	: "onInvalidHTTPMethod",
-						"GET"   	: "onInvalidHTTPMethod",
-						"POST"  	: "create",
-						"DELETE"	: "onInvalidHTTPMethod",
-						"PUT"   	: "create",
-						"PATCH" 	: "onInvalidHTTPMethod"
-					}
-				}
-			];
-		}
+		interceptorSettings = {
+			customInterceptionPoints : [
+				"writeToAppender"
+			]
+		};
 
     }
 
@@ -154,9 +144,17 @@ component {
 			// Register the appender with the root loggger, and turn the logger on.
 			var root = logBox.getRootLogger();
 			root.addAppender( appenders[ 'logstash_appender' ] );
-
 		}
 
+		settings.detachedAppenders.each( ( appender) => {
+			wirebox.getInstance( "AppenderService@cbelasticsearch" )
+				.createDetachedAppender(
+					appender.name,
+					appender.properties ?: {},
+					appender.class ?: "cbelasticsearch.models.logging.LogstashAppender"
+				);
+		} );
+
 		// If the api
 		if( settings.enableAPI ){
 			binder.map( "EventAppender@logstash" )

diff --git a/build/docs/Configuration.md b/build/docs/Configuration.md
@@ -56,7 +56,20 @@ moduleSettings = {
 		"indexPrefix"           : getSystemSetting( "LOGSTASH_INDEX_PREFIX", "" ),
 		"migrateIndices"  		: getSystemSetting( "LOGSTASH_MIGRATE_V2", false ),
 		// Whether to throw an error when a log document fails to save
-		"throwOnError"    		: true
+		"throwOnError"    		: true,
+		// An array of detached appenders which can be used with the `writeToAppender` interception point or directly through the elasticsearch module AppenderService	
+		"detachedAppenders"     : [
+			{
+				"name" : "myCustomAppender",
+				"properties" : {
+					"retentionDays"         : 1,
+					// The name of the application which will be transmitted with the log data and used for grouping
+					"applicationName"       : "Custom Detached Appender Logs",
+					// The max shard size at which the hot phase will rollover data
+					"rolloverSize"          : "1gb"
+				}
+			}
+		]
 	}
 }
 ```

diff --git a/build/docs/Logging.md b/build/docs/Logging.md
@@ -101,6 +101,25 @@ Fields typed `keyword` are not searchable, but are exact match fields.  This all
 
 _Note: There are two timestamp fields which contain the same data:  `timestamp` and `@timestamp`.  The latter is simply provided for easy automation with the default configuration for Logstash logs in Kibana.  [Read more on the ELK stack here](https://www.elastic.co/what-is/elk-stack)._
 
+## Detached Appender Logging
+
+If you [configure the `detachedAppenders` array](https://logstash.ortusbooks.com/getting-started/configuration) in your module config, you can log ad-hoc log messages directly to those data streams in Elasticsearch.   You may either [log through the `AppenderService` directly](https://cbelasticsearch.ortusbooks.com/logging) or the Logstash module provides a convenient interception point to log directly to your detached appenders.
+
+```java
+announce(
+	"writeToAppender", 
+	{
+		// The detached appender name
+		"name" : "myCustomAppender",
+		"level" : "info",
+		"message" : "Here is my custom log message",
+		"extraInfo" : {
+			"foo" : "bar"
+		}
+	}
+);
+```
+
 ## API Usage
 
 You may transmit any data directly to the logstash API, as long as it follows the mapped schema above.  You may even specify a name of the index prefix to be used in the transmission ( the actual index name will have the rotational appender timestamps applied, according to your configured rotation frequency).  This provides you flexibility in storing additional log files, which may or may not be from your CFML application. 

diff --git a/changelog.md b/changelog.md
@@ -6,64 +6,95 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ----
+
 ## [Unreleased]
+
 ### Changed
+
 * Remove JSONToRC module dependency as Coldbox handles this by default
 
 ## [3.0.4] => 2023-11-19
+
 ### Changed
+
 * Bump cbElasticsearch version to 3.2
+
 ## [3.0.3] => 2023-06-01
+
 ### Changed
+
 * Bump cbElasticsearch version to 3.1
 
 ## [3.0.2] => 2023-03-28
+
 ### Changed
+
 * Bump cbElasticsearch version to 3.0.2
 
 ## [3.0.1] => 2023-03-12
+
 ### Changed
+
 * Bump cbElasticsearch version to 3.0.1
 
 ## [3.0.0] => 2023-03-03
+
 ### Changed
+
 * Updates `cbElasticsearch` dependency for v3
 * Module settings and environment variable changes to support v3 of Logstash appender
 
 ## [2.0.1] => 2023-01-30
 
 ### Fixed
+
 * Revert to old log index pattern until data stream support can be implemented
 
 ## [2.0.0] => 2023-01-29
+
 ### Changed
+
 * Bump cbElasticsearch version minimum to 2.4.0
 * Bump Coldbox to v6
 * Changes default log index pattern to `logs-` to reflect v8 changes in Kibana/Logstash defaults
+
 ## [1.2.1] => 2022-09-21
+
 ### Added
+
 * Added support for `LOGSTASH_APPLICATION_NAME` environment variable
+
 ### Changed
+
 * Changed build process to use Github Actions
 * Migrated README content to GitBook
+
 ## [1.2.0] => 2022-08-12
+
 ### Fixed
+
 * Ensured SSL protocol on download location of package
 
 ## [1.1.1] => 2020-12-10
+
 ### Fixed
+
 * Remove build artifact from final package
 
 ## [1.1.0] => 2020-11-03
+
 ### Added
+
 * Added additional settings for number of index shards and replicas
 
 ### Changed
+
 * Modifies appender preflight to use base appender preflight
 * Modifies default logstash index prefix to use Kibana/ES conventions
 * Bumps `cbElasticsearch` dependency to `v2.1.0`
 
 ## [1.0.0] => 2020-09-11
 
 ### Added
-* Initial release of module
+
+* Initial release of module
diff --git a/config/Router.cfc b/config/Router.cfc
@@ -0,0 +1,21 @@
+component {
+
+	function configure(){
+		var apiEnabled = controller.getModuleSettings( "logstash", "apiEnabled", true );
+
+		if( apiEnabled ){
+			route( "/api/logs" )
+					.withAction( {
+						"HEAD"		: "onInvalidHTTPMethod",
+						"OPTIONS"	: "onInvalidHTTPMethod",
+						"GET"   	: "onInvalidHTTPMethod",
+						"POST"  	: "create",
+						"DELETE"	: "onInvalidHTTPMethod",
+						"PUT"   	: "create",
+						"PATCH" 	: "onInvalidHTTPMethod"
+					} )
+					.toHandler( "API" );
+		}
+	}
+
+}
diff --git a/interceptors/LogEvents.cfc b/interceptors/LogEvents.cfc
@@ -0,0 +1,25 @@
+component {
+
+	property name="appenderService" inject="AppenderService@cbelasticsearch";
+
+
+	function writeToAppender( event, rc, prc, interceptData ){
+		if( !interceptData.keyExists( "appender" ) ){
+			throw( type="InvalidArgument", message="The 'appender' key is required in the intercept data.  Could not continue." );
+		} else if( !interceptData.keyExists( "message" ) ){
+			throw( type="InvalidArgument", message="The 'message' key is required in the intercept data.  Could not continue." );
+		}
+
+		param interceptData.extraInfo = {};
+		param interceptData.severity = "INFO";
+
+		appenderService.logToAppender(
+			interceptData.appender,
+			interceptData.message,
+			interceptData.severity,
+			interceptData.extraInfo
+		);
+
+	}
+
+}
diff --git a/test-harness/[email protected] → [email protected] b/test-harness/[email protected] → [email protected]
@@ -10,7 +10,8 @@
         },
         "rewrites":{
             "enable":"true"
-        }
+        },
+        "webroot":"test-harness"
     },
     "openBrowser":"false"
-}
+}
diff --git a/test-harness/[email protected] → [email protected] b/test-harness/[email protected] → [email protected]
@@ -1,5 +1,5 @@
 {
-    "name":"logstash-adobe@2018",
+    "name":"logstash-adobe@2021",
     "app":{
         "serverHomeDirectory":".engine/adobe2021",
         "cfengine":"adobe@2021"
@@ -11,13 +11,14 @@
         "rewrites":{
             "enable":"true"
         },
+        "webroot":"test-harness",
         "aliases":{
-            "/moduleroot/stachebox/includes":"../includes",
-            "/stachebox/includes":"../includes"
+            "/moduleroot/logstash":"../",
+            "/root":"./test-harness"
         }
     },
     "openBrowser":"false",
-	"scripts" : {
-		"onServerInstall" : "cfpm install zip,mysql,document,feed"
-	}
-}
+    "scripts":{
+        "onServerInstall":"cfpm install zip,debugger"
+    }
+}
diff --git a/test-harness/[email protected] → [email protected] b/test-harness/[email protected] → [email protected]
@@ -1,7 +1,7 @@
 {
     "name":"logstash-adobe@2023",
     "app":{
-        "serverHomeDirectory":"./.engine/adobe2023",
+        "serverHomeDirectory":".engine/adobe2023",
         "cfengine":"adobe@2023"
     },
     "web":{
@@ -11,13 +11,14 @@
         "rewrites":{
             "enable":"true"
         },
+        "webroot":"test-harness",
         "aliases":{
-            "/moduleroot/stachebox/includes":"../includes",
-            "/stachebox/includes":"../includes"
+            "/moduleroot/logstash":"../",
+            "/root":"./test-harness"
         }
     },
     "openBrowser":"false",
     "scripts":{
-        "onServerInstall":"cfpm install zip,document,feed"
+        "onServerInstall":"cfpm install zip,debugger"
     }
-}
+}
diff --git a/test-harness/[email protected] → [email protected] b/test-harness/[email protected] → [email protected]
@@ -10,7 +10,12 @@
         },
         "rewrites":{
             "enable":"true"
+        },
+        "webroot":"test-harness",
+        "aliases":{
+            "/moduleroot/logstash":"./",
+            "/root":"./test-harness"
         }
     },
     "openBrowser":"false"
-}
+}