From c810a9bde7a9e2e306e70c3ec0123504f2907a4a Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:07:21 -0600 Subject: [PATCH] [citrix_adc] Improve timestamp parsing (#11698) - Improve parsing of various timestamps and improve handling of custom date formats. - Custom date formats are now handled first, falling back to a standard set of patterns using a date processor otherwise. - Consolidated parsing of timestamps to default pipeline. --- .../deploy/docker/sample_logs/citrix-adc.log | 1 - packages/citrix_adc/changelog.yml | 5 + ...trix-native-with-delink.json-expected.json | 10 +- .../test/pipeline/test-citrix-native.json | 4 + .../test-citrix-native.json-config.yml | 6 + .../test-citrix-native.json-expected.json | 162 +++++++++++--- .../elasticsearch/ingest_pipeline/default.yml | 203 +++++++++++++++--- .../ingest_pipeline/ica_feature.yml | 27 +-- .../sslvpn_and_aaatm_feature.yml | 77 ++----- .../ingest_pipeline/tcp_and_acl_feature.yml | 90 +------- packages/citrix_adc/manifest.yml | 2 +- 11 files changed, 345 insertions(+), 242 deletions(-) create mode 100644 packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-config.yml diff --git a/packages/citrix_adc/_dev/deploy/docker/sample_logs/citrix-adc.log b/packages/citrix_adc/_dev/deploy/docker/sample_logs/citrix-adc.log index 33b518c8e0a..968dc8370ec 100644 --- a/packages/citrix_adc/_dev/deploy/docker/sample_logs/citrix-adc.log +++ b/packages/citrix_adc/_dev/deploy/docker/sample_logs/citrix-adc.log @@ -15,4 +15,3 @@ Oct 6 14:03:30 81.2.69.144 10/06/2014:14:03:30 GMT ns1 0-PPE-0 : T Oct 6 14:03:30 81.2.69.144 10/06/2014:14:03:30 GMT ns1 0-PPE-0 : TCP CONN_TERMINATE 4473 0 : Source 127.0.0.1:7776 - Destination 127.0.0.2:55623 - Start Time 10/06/2014:14:02:45 GMT - End Time 10/06/2014:14:03:30 GMT - Total_bytes_send 1 - Total_bytes_recv 1 Oct 6 14:03:30 81.2.69.144 10/06/2014:14:03:30 GMT ns1 0-PPE-0 : TCP CONN_TERMINATE 4474 0 : Source 127.0.0.1:80 - Destination 127.0.0.2:39771 - Start Time 10/06/2014:14:02:46 GMT - End Time 10/06/2014:14:03:30 GMT - Total_bytes_send 1 - Total_bytes_recv 1 <131> 10/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 : Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 10/08/2024:09:38:41 - Total_bytes_send 0 - Total_bytes_recv 3118 -<131> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 : Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 10/08/2024:09:38:41 - Total_bytes_send 0 - Total_bytes_recv 3118 \ No newline at end of file diff --git a/packages/citrix_adc/changelog.yml b/packages/citrix_adc/changelog.yml index e64dad04868..3dc7abd93a3 100644 --- a/packages/citrix_adc/changelog.yml +++ b/packages/citrix_adc/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: "Improve timestamp parsing" + type: bugfix + link: https://github.com/elastic/integrations/pull/11698 - version: "1.10.0" changes: - description: "Parse additional sslvpn fields" diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json index de67bfe0e38..d42deddd940 100644 --- a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native-with-delink.json-expected.json @@ -45,7 +45,7 @@ ] }, { - "@timestamp": "2024-08-10:38:41.000Z", + "@timestamp": "2024-08-10T09:38:41.000Z", "citrix": { "cef_format": false, "default_class": true, @@ -59,7 +59,7 @@ }, "citrix_adc": { "log": { - "delink_time": "2024-08-10T09:38:41", + "delink_time": "2024-08-10T09:38:41.000Z", "destination": { "ip": "81.2.69.144", "port": 80 @@ -93,7 +93,7 @@ "category": [ "network" ], - "end": "2024-08-10T09:38:41", + "end": "2024-08-10T09:38:41.000Z", "id": "6715345", "original": "<131> 10/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 : Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 10/08/2024:09:38:41 - Total_bytes_send 0 - Total_bytes_recv 3118\n", "severity": 0, @@ -192,7 +192,7 @@ }, "citrix_adc": { "log": { - "delink_time": "2024-08-21T09:38:41", + "delink_time": "2024-08-21T09:38:41.000Z", "destination": { "ip": "81.2.69.144", "port": 80 @@ -226,7 +226,7 @@ "category": [ "network" ], - "end": "2024-08-21T09:38:41", + "end": "2024-08-21T09:38:41.000Z", "id": "6715345", "original": "<131> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 : Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 21/08/2024:09:38:41 - Total_bytes_send 0 - Total_bytes_recv 3118\n", "severity": 0, diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json index 90e5b7a8392..a14d2d7a1aa 100644 --- a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json @@ -75,6 +75,10 @@ { "@timestamp": "2024-08-21T09:38:41.000Z", "message": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN CHANGEME 600000 0 : \n" + }, + { + "@timestamp": "2024-08-21T09:38:41.000Z", + "message": "<134> 21/08/2024:13:25:41 SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 \n" } ] } diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-config.yml b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-config.yml new file mode 100644 index 00000000000..a483e20a611 --- /dev/null +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-config.yml @@ -0,0 +1,6 @@ +fields: + tags: + - preserve_original_event + - preserve_duplicate_custom_fields + _conf: + custom_date_format: "dd/MM/yyyy:HH:mm:ss" diff --git a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-expected.json b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-expected.json index ee3844f6add..6265f2c0c7b 100644 --- a/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-expected.json +++ b/packages/citrix_adc/data_stream/log/_dev/test/pipeline/test-citrix-native.json-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2024-10-08T09:38:41.000Z", + "@timestamp": "2024-08-10T09:38:41.000Z", "citrix": { "cef_format": false, "default_class": true, @@ -15,7 +15,7 @@ }, "citrix_adc": { "log": { - "delink_time": "2024-10-08T09:38:41.000Z", + "delink_time": "2024-08-10T09:38:41.000Z", "destination": { "ip": "81.2.69.144", "port": 80 @@ -49,7 +49,7 @@ "category": [ "network" ], - "end": "2024-10-08T09:38:41.000Z", + "end": "2024-08-10T09:38:41.000Z", "id": "6715345", "original": "<131> 10/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default TCP CONN_DELINK 6715345 0 : Source 127.1.2.1:80 - Vserver 81.2.69.144:80 - NatIP 192.168.10.10:52187 - Destination 81.2.69.144:80 - Delink Time 10/08/2024:09:38:41 - Total_bytes_send 0 - Total_bytes_recv 3118\n", "severity": 0, @@ -90,7 +90,7 @@ ] }, { - "@timestamp": "2024-10-08T09:38:41.000Z", + "@timestamp": "2024-08-10T09:38:41.000Z", "citrix": { "cef_format": false, "default_class": true, @@ -244,7 +244,7 @@ "port": 443 }, "duration": "00:00:04 ", - "end_time": "2024-11-06T08:33:03.000Z", + "end_time": "2024-06-11T08:33:03.000Z", "groups": "N/A", "message": "Context user_name@domain.com@0.0.0.0 - SessionId: 343368 - User user_name - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:51607 - Destination 0.0.0.0:443 - Start_time \"11/06/2024:08:32:59\" - End_time \"11/06/2024:08:33:03\" - Duration 00:00:04 - Total_bytes_send 0 - Total_bytes_recv 378 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\" ", "nat": { @@ -255,7 +255,7 @@ "ip": "0.0.0.0", "port": 51607 }, - "start_time": "2024-11-06T08:32:59.000Z", + "start_time": "2024-06-11T08:32:59.000Z", "total_bytes_received": 378, "total_bytes_send": 0, "total_compressed_bytes_recieved": 0, @@ -283,11 +283,11 @@ "category": [ "authentication" ], - "end": "2024-11-06T08:33:03.000Z", + "end": "2024-06-11T08:33:03.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN TCPCONNSTAT 600000 0 : Context user_name@domain.com@0.0.0.0 - SessionId: 343368 - User user_name - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:51607 - Destination 0.0.0.0:443 - Start_time \"11/06/2024:08:32:59\" - End_time \"11/06/2024:08:33:03\" - Duration 00:00:04 - Total_bytes_send 0 - Total_bytes_recv 378 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\" \n", "severity": 0, - "start": "2024-11-06T08:32:59.000Z", + "start": "2024-06-11T08:32:59.000Z", "timezone": "UTC", "type": [ "info" @@ -354,7 +354,7 @@ }, "session_id": "342014", "sso_status": "ON", - "timestamp": "2024-11-06T08:33:03.000Z", + "timestamp": "2024-06-11T08:33:03.000Z", "user": "user_name", "username": "user_name@domain.com", "vserver": { @@ -439,7 +439,7 @@ "ip": "0.0.0.0", "port": 62480 }, - "start_time": "2024-11-06T08:32:58.000Z", + "start_time": "2024-06-11T08:32:58.000Z", "username": "username" } }, @@ -457,7 +457,7 @@ "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN ICASTART 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 0.0.0.0:62480 - Destination 0.0.0.0:2598 - customername - username:domainname username:domain - applicationName Developer Europe $P14189 - startTime \"11/06/2024:08:32:58\" - connectionId 16879892 \n", "severity": 0, - "start": "2024-11-06T08:32:58.000Z", + "start": "2024-06-11T08:32:58.000Z", "timezone": "UTC", "type": [ "info" @@ -513,13 +513,13 @@ }, "domain_name": "domain", "duration": "04:07:08", - "end_time": "2024-11-06T08:33:02.000Z", + "end_time": "2024-06-11T08:33:02.000Z", "message": "[TCP] [CGP][ICAUUID=000bb24f-efd3-172a-9678-000d3ac7ec06] Source 0.0.0.0:51547 - Destination 0.0.0.0:2598 - customername - username:domainname username:domain - startTime \"11/06/2024:04:25:54\" - endTime \"11/06/2024:08:33:02\" - Duration 04:07:08 - Total_bytes_send 109566281 - Total_bytes_recv 32996419 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 1057494 ", "source": { "ip": "0.0.0.0", "port": 51547 }, - "start_time": "2024-11-06T04:25:54.000Z", + "start_time": "2024-06-11T04:25:54.000Z", "total_bytes_received": 32996419, "total_bytes_send": 109566281, "total_compressed_bytes_recieved": 0, @@ -539,11 +539,11 @@ "category": [ "authentication" ], - "end": "2024-11-06T08:33:02.000Z", + "end": "2024-06-11T08:33:02.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=000bb24f-efd3-172a-9678-000d3ac7ec06] Source 0.0.0.0:51547 - Destination 0.0.0.0:2598 - customername - username:domainname username:domain - startTime \"11/06/2024:04:25:54\" - endTime \"11/06/2024:08:33:02\" - Duration 04:07:08 - Total_bytes_send 109566281 - Total_bytes_recv 32996419 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 1057494 \n", "severity": 0, - "start": "2024-11-06T04:25:54.000Z", + "start": "2024-06-11T04:25:54.000Z", "timezone": "UTC", "type": [ "info" @@ -670,14 +670,14 @@ "compression_ratio_recieved": 0.0, "compression_ratio_send": 0.0, "duration": "00:43:40 ", - "end_time": "2024-11-06T08:32:57.000Z", + "end_time": "2024-06-11T08:32:57.000Z", "groups": "N/A", "http_resources_accessed": "0", "logout_method": "TimedOut", "message": "Logout handler : Context user_name@domain.com@0.0.0.0 - SessionId: 17790 - User user_name - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/06/2024:07:49:17\" - End_time \"11/06/2024:08:32:57\" - Duration 00:43:40 - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 176 - Total_UDP_flows 0 - Total_policies_allowed 175 - Total_policies_denied 0 - Total_bytes_send 804 - Total_bytes_recv 3079180 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\" ", "non_http_services_accessed": "0", "session_id": "17790", - "start_time": "2024-11-06T07:49:17.000Z", + "start_time": "2024-06-11T07:49:17.000Z", "total_bytes_received": 3079180, "total_bytes_send": 804, "total_compressed_bytes_recieved": 0, @@ -707,11 +707,11 @@ "category": [ "authentication" ], - "end": "2024-11-06T08:32:57.000Z", + "end": "2024-06-11T08:32:57.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Logout handler : Context user_name@domain.com@0.0.0.0 - SessionId: 17790 - User user_name - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/06/2024:07:49:17\" - End_time \"11/06/2024:08:32:57\" - Duration 00:43:40 - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 176 - Total_UDP_flows 0 - Total_policies_allowed 175 - Total_policies_denied 0 - Total_bytes_send 804 - Total_bytes_recv 3079180 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\" \n", "severity": 0, - "start": "2024-11-06T07:49:17.000Z", + "start": "2024-06-11T07:49:17.000Z", "timezone": "UTC", "type": [ "info" @@ -881,7 +881,7 @@ "port": 443 }, "duration": "00:00:00 ", - "end_time": "2024-11-08T15:01:39.000Z", + "end_time": "2024-08-11T15:01:39.000Z", "groups": "N/A", "message": "Context user_name@acme.com@0.0.0.0 - SessionId: 346153 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:4595 - Destination 0.0.0.0:443 - Start_time \"11/08/2024:15:01:39\" - End_time \"11/08/2024:15:01:39\" - Duration 00:00:00 - Total_bytes_send 0 - Total_bytes_recv 417 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\"", "nat": { @@ -892,7 +892,7 @@ "ip": "0.0.0.0", "port": 4595 }, - "start_time": "2024-11-08T15:01:39.000Z", + "start_time": "2024-08-11T15:01:39.000Z", "total_bytes_received": 417, "total_bytes_send": 0, "total_compressed_bytes_recieved": 0, @@ -920,11 +920,11 @@ "category": [ "authentication" ], - "end": "2024-11-08T15:01:39.000Z", + "end": "2024-08-11T15:01:39.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN TCPCONNSTAT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 346153 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip 0.0.0.0 - Vserver 0.0.0.0:443 - Source 0.0.0.0:4595 - Destination 0.0.0.0:443 - Start_time \"11/08/2024:15:01:39\" - End_time \"11/08/2024:15:01:39\" - Duration 00:00:00 - Total_bytes_send 0 - Total_bytes_recv 417 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - Access Allowed - Group(s) \"N/A\"\n", "severity": 0, - "start": "2024-11-08T15:01:39.000Z", + "start": "2024-08-11T15:01:39.000Z", "timezone": "UTC", "type": [ "info" @@ -1060,14 +1060,14 @@ "compression_ratio_recieved": 0.0, "compression_ratio_send": 0.0, "duration": "00:32:13 ", - "end_time": "2024-11-08T11:46:32.000Z", + "end_time": "2024-08-11T11:46:32.000Z", "groups": "N/A", "http_resources_accessed": "0", "logout_method": "TimedOut", "message": "Context user_name@acme.com@0.0.0.0 - SessionId: 352037 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:11:14:19\" - End_time \"11/08/2024:11:46:32\" - Duration 00:32:13 - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 67 - Total_UDP_flows 0 - Total_policies_allowed 67 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 1529833 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\"", "non_http_services_accessed": "0", "session_id": "352037", - "start_time": "2024-11-08T11:14:19.000Z", + "start_time": "2024-08-11T11:14:19.000Z", "total_bytes_received": 1529833, "total_bytes_send": 0, "total_compressed_bytes_recieved": 0, @@ -1097,11 +1097,11 @@ "category": [ "authentication" ], - "end": "2024-11-08T11:46:32.000Z", + "end": "2024-08-11T11:46:32.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 352037 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:11:14:19\" - End_time \"11/08/2024:11:46:32\" - Duration 00:32:13 - Http_resources_accessed 0 - NonHttp_services_accessed 0 - Total_TCP_connections 67 - Total_UDP_flows 0 - Total_policies_allowed 67 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 1529833 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"TimedOut\" - Group(s) \"N/A\"\n", "severity": 0, - "start": "2024-11-08T11:14:19.000Z", + "start": "2024-08-11T11:14:19.000Z", "timezone": "UTC", "type": [ "info" @@ -1157,14 +1157,14 @@ "compression_ratio_recieved": 0.0, "compression_ratio_send": 0.0, "duration": "00:06:26 ", - "end_time": "2024-11-08T10:43:39.000Z", + "end_time": "2024-08-11T10:43:39.000Z", "groups": "N/A", "http_resources_accessed": "7", "logout_method": "Explicit", "message": "Context user_name@acme.com@0.0.0.0 - SessionId: 351869 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:10:37:13\" - End_time \"11/08/2024:10:43:39\" - Duration 00:06:26 - Http_resources_accessed 7 - NonHttp_services_accessed 0 - Total_TCP_connections 14 - Total_UDP_flows 0 - Total_policies_allowed 14 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 86130 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"Explicit\" - Group(s) \"N/A\"", "non_http_services_accessed": "0", "session_id": "351869", - "start_time": "2024-11-08T10:37:13.000Z", + "start_time": "2024-08-11T10:37:13.000Z", "total_bytes_received": 86130, "total_bytes_send": 0, "total_compressed_bytes_recieved": 0, @@ -1194,11 +1194,11 @@ "category": [ "authentication" ], - "end": "2024-11-08T10:43:39.000Z", + "end": "2024-08-11T10:43:39.000Z", "id": "600000", "original": "<123> 21/08/2024:09:38:41 SYSLOGHOST 0-PPE-1 : default SSLVPN LOGOUT 600000 0 : Context user_name@acme.com@0.0.0.0 - SessionId: 351869 - User user_name@acme.com - Client_ip 0.0.0.0 - Nat_ip \"Mapped Ip\" - Vserver 0.0.0.0:443 - Start_time \"11/08/2024:10:37:13\" - End_time \"11/08/2024:10:43:39\" - Duration 00:06:26 - Http_resources_accessed 7 - NonHttp_services_accessed 0 - Total_TCP_connections 14 - Total_UDP_flows 0 - Total_policies_allowed 14 - Total_policies_denied 0 - Total_bytes_send 0 - Total_bytes_recv 86130 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - LogoutMethod \"Explicit\" - Group(s) \"N/A\"\n", "severity": 0, - "start": "2024-11-08T10:37:13.000Z", + "start": "2024-08-11T10:37:13.000Z", "timezone": "UTC", "type": [ "info" @@ -1339,6 +1339,106 @@ "preserve_original_event", "preserve_duplicate_custom_fields" ] + }, + { + "@timestamp": "2024-08-21T13:25:41.000Z", + "citrix": { + "cef_format": false, + "default_class": true, + "detail": "<134> 21/08/2024:13:25:41 SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 ", + "device_event_class_id": "SSLVPN", + "extended": { + "message": "[TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 " + }, + "host": "SYSLOGHOST", + "name": "ICAEND_CONNSTAT" + }, + "citrix_adc": { + "log": { + "compression_ratio_recieved": 0.0, + "compression_ratio_send": 0.0, + "connection_id": "20459456", + "destination": { + "ip": "10.0.10.75", + "port": 2598 + }, + "domain_name": "domain_name", + "duration": "00:06:26", + "end_time": "2024-08-11T10:43:39.000Z", + "message": "[TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 ", + "source": { + "ip": "67.43.156.1", + "port": 50385 + }, + "start_time": "2024-08-11T10:37:13.000Z", + "total_bytes_received": 2761789, + "total_bytes_send": 8379078, + "total_compressed_bytes_recieved": 0, + "total_compressed_bytes_send": 0, + "username": "user_name" + } + }, + "destination": { + "bytes": 2761789, + "ip": "10.0.10.75", + "port": 2598 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "authentication" + ], + "end": "2024-08-11T10:43:39.000Z", + "id": "600000", + "original": "<134> 21/08/2024:13:25:41 SYSLOGHOST 0-PPE-1 : default SSLVPN ICAEND_CONNSTAT 600000 0 : [TCP] [CGP][ICAUUID=00033ef4-29bb-172b-9678-0022480fced0] Source 67.43.156.1:50385 - Destination 10.0.10.75:2598 - customername - username:domainname user_name:domain_name - startTime \"11/08/2024:10:37:13 \" - endTime \"11/08/2024:10:43:39 \" - Duration 00:06:26 - Total_bytes_send 8379078 - Total_bytes_recv 2761789 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 20459456 \n", + "severity": 0, + "start": "2024-08-11T10:37:13.000Z", + "timezone": "UTC", + "type": [ + "info" + ] + }, + "observer": { + "product": "Netscaler", + "type": "firewall", + "vendor": "Citrix" + }, + "related": { + "ip": [ + "67.43.156.1", + "10.0.10.75" + ], + "user": [ + "user_name" + ] + }, + "source": { + "as": { + "number": 35908 + }, + "bytes": 8379078, + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + } + }, + "ip": "67.43.156.1", + "port": 50385 + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "user": { + "domain": "domain_name", + "name": "user_name" + } } ] } \ No newline at end of file diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 13b588b8449..325524359ab 100644 --- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -33,7 +33,7 @@ processors: pattern_definitions: LEVEL: '?' IDENT: '[a-zA-Z][a-zA-Z0-9]*' - SYSLOG_TIMESTAMP: '(?:%{SYSLOGTIMESTAMP:_tmp.timestamp}|%{TIMESTAMP_ISO8601:_tmp.timestamp8601})' + SYSLOG_TIMESTAMP: '(?:%{SYSLOGTIMESTAMP:_tmp.syslog_timestamp}|%{TIMESTAMP_ISO8601:_tmp.syslog_timestamp8601})' TIMESTAMP_ISO8601: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?' - pipeline: name: '{{ IngestPipeline "cef" }}' @@ -48,6 +48,9 @@ processors: tag: convert_event_severity_to_long type: long ignore_missing: true + +# Time zone + - set: field: _conf.tz_offset tag: set_tz_offset_utc @@ -58,22 +61,33 @@ processors: tag: set_event_timezone_from_tz_offset copy_from: _conf.tz_offset if: ctx.event?.timezone == null || ctx.event?.timezone == "" + +# Syslog timestamp + - date: if: ctx._tmp?.timestamp8601 != null - tag: date_tmp_timestamp8601 - field: _tmp.timestamp8601 + tag: date_syslog_timestamp8601 + field: _tmp.syslog_timestamp8601 timezone: '{{{event.timezone}}}' formats: - ISO8601 + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: - field: _tmp.timestamp - tag: set_tmp_timestamp - if: ctx._tmp?.timestamp != null && ctx.citrix?.event_year != null - value: "{{{citrix.event_year}}} {{{_tmp.timestamp}}}" + field: _tmp.syslog_timestamp + tag: enrich_syslog_timestamp_with_year + if: ctx._tmp?.syslog_timestamp != null && ctx.citrix?.event_year != null + value: "{{{citrix.event_year}}} {{{_tmp.syslog_timestamp}}}" + - remove: + field: citrix.event_year + tag: remove_event_year + ignore_missing: true - date: - if: ctx._tmp?.timestamp != null - tag: date_tmp_timestamp - field: _tmp.timestamp + if: ctx._tmp?.syslog_timestamp != null + tag: date_syslog_timestamp + field: _tmp.syslog_timestamp timezone: '{{{event.timezone}}}' formats: - MMM d HH:mm:ss @@ -88,35 +102,168 @@ processors: - yyyy MMMM d HH:mm:ss - yyyy MMMM d HH:mm:ss - yyyy MMMM dd HH:mm:ss + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + - script: + tag: trim_time_fields + lang: painless + params: + fields: + - timestamp_native + - timestamp + - delink_time + - start_time + - end_time + source: |- + params.fields.forEach(field -> { + if (!ctx._tmp?.containsKey(field) || !(ctx._tmp[field] instanceof String)) { + return true; + } + + String val = ctx._tmp[field]; + ctx._tmp[field] = val.trim(); + }); + - script: - description: Convert timestamp_native via custom format - tag: date_tmp_timestamp_native_custom_format + tag: parse_custom_time_formats + if: ctx._conf?.custom_date_format != null lang: painless - source: >- - def dateFormat = ctx["_conf"]["custom_date_format"]; - def formatter = DateTimeFormatter.ofPattern(dateFormat); - def text = ctx["_tmp"]["timestamp_native"]; - def parsedDate = LocalDateTime.parse(text, formatter); - ctx["_tmp"]["timestamp_native"] = parsedDate.toString(); - - if: ctx._tmp?.timestamp_native != null && ctx?._conf?.custom_date_format != null && ctx?._conf?.custom_date_format != '' + params: + fields: + - timestamp_native + - timestamp + - delink_time + - start_time + - end_time + source: |- + def zone = ctx.event?.timezone != null ? ZoneId.of(ctx.event.timezone) : null; + def formatter = DateTimeFormatter.ofPattern(ctx._conf.custom_date_format); + def outFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + + params.fields.forEach(field -> { + if (!ctx._tmp?.containsKey(field)) { + return true; + } + + try { + def localDateTime = LocalDateTime.parse(ctx._tmp[field], formatter); + ctx.citrix_adc.log[field] = outFormatter.format(ZonedDateTime.of(localDateTime, zone)); + } catch (Exception e) { + /* Intentionally ignored */ + return true; + } + }); + +# Native-format timestamp + + - date: + tag: date_timestamp_native + field: _tmp.timestamp_native + target_field: citrix_adc.log.timestamp_native + if: ctx._tmp?.timestamp_native != null && ctx.citrix_adc?.log?.timestamp_native == null + formats: + - ISO8601 + - yyyy/MM/dd:HH:mm:ss + - yyyy/MM/dd:HH:mm:ss z + - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z on_failure: - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}. Offending format: {{{_conf.custom_date_format}}}.' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: - if: ctx._tmp?.timestamp_native != null && ctx?._conf?.custom_date_format == null - tag: date_tmp_timestamp_native - field: _tmp.timestamp_native + tag: date_timestamp + field: _tmp.timestamp + target_field: citrix_adc.log.timestamp + if: ctx._tmp?.timestamp != null && ctx.citrix_adc?.log?.timestamp == null formats: + - ISO8601 + - yyyy/MM/dd:HH:mm:ss + - yyyy/MM/dd:HH:mm:ss z - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + tag: date_start_time + field: _tmp.start_time + target_field: citrix_adc.log.start_time + if: ctx._tmp?.start_time != null && ctx.citrix_adc?.log?.start_time == null + formats: + - ISO8601 - yyyy/MM/dd:HH:mm:ss - - dd/MM/yyyy:HH:mm:ss - timezone: '{{{event.timezone}}}' + - yyyy/MM/dd:HH:mm:ss z + - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + tag: date_end_time + field: _tmp.end_time + target_field: citrix_adc.log.end_time + if: ctx._tmp?.end_time != null && ctx.citrix_adc?.log?.end_time == null + formats: + - ISO8601 + - yyyy/MM/dd:HH:mm:ss + - yyyy/MM/dd:HH:mm:ss z + - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + tag: date_delink_time + field: _tmp.delink_time + target_field: citrix_adc.log.delink_time + if: ctx._tmp?.delink_time != null && ctx.citrix_adc?.log?.delink_time == null + formats: + - ISO8601 + - yyyy/MM/dd:HH:mm:ss + - yyyy/MM/dd:HH:mm:ss z + - MM/dd/yyyy:HH:mm:ss + - MM/dd/yyyy:HH:mm:ss z + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + +# Move vendor time fields to ECS. + + - set: + tag: set_@timestamp_from_citrix_native + field: '@timestamp' + copy_from: citrix_adc.log.timestamp_native + ignore_empty_value: true - remove: - field: citrix.event_year - tag: remove_event_year + tag: remove_citrix_timestamp_native + field: citrix_adc.log.timestamp_native ignore_missing: true + + - set: + tag: set_event_start_from_citrix_start_time + field: event.start + copy_from: citrix_adc.log.start_time + ignore_empty_value: true + + - set: + tag: set_event_end_from_citrix_delink_time + field: event.end + copy_from: citrix_adc.log.delink_time + ignore_empty_value: true + - set: + tag: set_event_end_from_citrix_end_time + field: event.end + copy_from: citrix_adc.log.end_time + ignore_empty_value: true + - geoip: field: client.ip tag: geoip_client_ip_to_client_geo diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ica_feature.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ica_feature.yml index 1c3b2c13034..e30694b2271 100644 --- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ica_feature.yml +++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/ica_feature.yml @@ -8,7 +8,7 @@ processors: patterns: - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie%{SPACE}:%{SPACE}%{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - session_setup_time %{DATA:citrix_adc.log.session_setup_time} - client_ip %{IP:citrix_adc.log.client_ip} - client_type %{NUMBER:citrix_adc.log.client_type:int} - client_launcher %{NUMBER:citrix_adc.log.client_launcher:int} - client_version %{DATA:citrix_adc.log.client_version} - client_hostname %{DATA:citrix_adc.log.client_hostname} - domain_name %{DATA:citrix_adc.log.domain_name} - server_name %{DATA:citrix_adc.log.server.name} - connection_priority %{NUMBER:citrix_adc.log.connection_priority:int} - access_type %{NUMBER:citrix_adc.log.access_type:int} - status %{NUMBER:citrix_adc.log.status:int} - username %{USERNAME:citrix_adc.log.username}$' - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - channel_update_begin %{DATA:citrix_adc.log.channel_update.begin} - channel_update_end %{DATA:citrix_adc.log.channel_update.end} - channel_id_1 %{NUMBER:citrix_adc.log.channel_id_1:int} - channel_id_1_val %{NUMBER:citrix_adc.log.channel_id_1_val:int} - channel_id_2 %{NUMBER:citrix_adc.log.channel_id_2:int} - channel_id_2_val %{NUMBER:citrix_adc.log.channel_id_2_val:int} - channel_id_3 %{NUMBER:citrix_adc.log.channel_id_3:int} - channel_id_3_val %{NUMBER:citrix_adc.log.channel_id_3_val:int} - channel_id_4 %{NUMBER:citrix_adc.log.channel_id_4:int} - channel_id_4_val %{NUMBER:citrix_adc.log.channel_id_4_val:int} - channel_id_5 %{NUMBER:citrix_adc.log.channel_id_5:int} - channel_id_5_val %{NUMBER:citrix_adc.log.channel_id_5_val:int}$' - - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - nsica_session_status %{NUMBER:citrix_adc.log.nsica_session.status:int} - nsica_session_client_ip %{IP:citrix_adc.log.nsica_session.client.ip} - nsica_session_client_port %{NUMBER:citrix_adc.log.nsica_session.client.port:int} - nsica_session_server_ip %{IP:citrix_adc.log.nsica_session.server.ip} - nsica_session_server_port %{NUMBER:citrix_adc.log.nsica_session.server.port:int} - nsica_session_reconnect_count %{NUMBER:citrix_adc.log.nsica_session.reconnect_count:int} - nsica_session_acr_count %{NUMBER:citrix_adc.log.nsica_session.acr_count:int} - connection_priority %{NUMBER:citrix_adc.log.connection_priority:int} - timestamp %{DATA:citrix_adc.log.timestamp} -$' + - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - nsica_session_status %{NUMBER:citrix_adc.log.nsica_session.status:int} - nsica_session_client_ip %{IP:citrix_adc.log.nsica_session.client.ip} - nsica_session_client_port %{NUMBER:citrix_adc.log.nsica_session.client.port:int} - nsica_session_server_ip %{IP:citrix_adc.log.nsica_session.server.ip} - nsica_session_server_port %{NUMBER:citrix_adc.log.nsica_session.server.port:int} - nsica_session_reconnect_count %{NUMBER:citrix_adc.log.nsica_session.reconnect_count:int} - nsica_session_acr_count %{NUMBER:citrix_adc.log.nsica_session.acr_count:int} - connection_priority %{NUMBER:citrix_adc.log.connection_priority:int} - timestamp %{DATA:_tmp.timestamp} -$' - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - nsica_status %{NUMBER:citrix_adc.log.nsica_status:int} - L7LatencyThresholdFactor %{NUMBER:citrix_adc.log.l7_latency.threshold_factor:int} - L7LatencyWaittime %{NUMBER:citrix_adc.log.l7_latency.waittime:int} - L7LatencyNotifyInterval %{NUMBER:citrix_adc.log.l7_latency.notify_interval:int} - L7LatencyMaxNotifyCount %{NUMBER:citrix_adc.log.l7_latency.max_notify_count:int} - L7ThresholdBreachAvgClientsideLatency %{NUMBER:citrix_adc.log.l7_threshold_breach.avg_clientside_latency:int} - L7ThresholdBreachMaxClientsideLatency %{NUMBER:citrix_adc.log.l7_threshold_breach.max_clientside_latency:int} - L7ThresholdBreachAvgServersideLatency %{NUMBER:citrix_adc.log.l7_threshold_breach.avg_serverside_latency:int} - L7ThresholdBreachMaxServersideLatency %{NUMBER:citrix_adc.log.l7_threshold_breach.max_serverside_latency:int} - MinL7Latency %{NUMBER:citrix_adc.log.min_l7_latency:int} -$' - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - session_end_time %{DATA:citrix_adc.log.session_end_time}$' - '^session_guid %{WORD:citrix_adc.log.session_guid} - device_serial_number %{NUMBER:citrix_adc.log.device_serial_number:int} - client_cookie %{WORD:citrix_adc.log.client_cookie} - flags %{NUMBER:citrix_adc.log.flags:int} - ica_rtt %{NUMBER:citrix_adc.log.ica_rtt:int} - clientside_rxbytes %{NUMBER:citrix_adc.log.clientside.rxbytes:int} - clientside_txbytes %{NUMBER:citrix_adc.log.clientside.txbytes:int} - clientside_packet_retransmits %{NUMBER:citrix_adc.log.clientside.packet_retransmits:int} - serverside_packet_retransmits %{NUMBER:citrix_adc.log.serverside.packet_retransmits:int} - clientside_rtt %{NUMBER:citrix_adc.log.clientside.rtt:int} - serverside_rtt %{NUMBER:citrix_adc.log.serverside.rtt:int} - clientside_jitter %{NUMBER:citrix_adc.log.clientside.jitter:int} - serverside_jitter %{NUMBER:citrix_adc.log.serverside.jitter:int}$' @@ -31,31 +31,6 @@ processors: tag: set_client_ip_from_client_ip copy_from: citrix_adc.log.client_ip ignore_empty_value: true - - date: - field: citrix_adc.log.timestamp - tag: date_timestamp - target_field: citrix_adc.log.timestamp - formats: - - UNIX - - ISO8601 - - MM/dd/yyyy:HH:mm:ss - - MMM d HH:mm:ss - - MMM d HH:mm:ss - - MMM dd HH:mm:ss - - MMMM d HH:mm:ss - - MMMM d HH:mm:ss - - MMMM dd HH:mm:ss - - yyyy MMM d HH:mm:ss - - yyyy MMM d HH:mm:ss - - yyyy MMM dd HH:mm:ss - - yyyy MMMM d HH:mm:ss - - yyyy MMMM d HH:mm:ss - - yyyy MMMM dd HH:mm:ss - if: ctx.citrix_adc?.log?.timestamp != null && ctx.citrix_adc.log.timestamp != '' - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: citrix_adc.log.clientside.rxbytes tag: convert_clientside_rxbytes_to_long diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml index e4bd6f09f7b..e438824523a 100644 --- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml +++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/sslvpn_and_aaatm_feature.yml @@ -16,32 +16,32 @@ processors: if: 'ctx.citrix.name == "LOGOUT"' field: citrix.extended.message patterns: - - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time "%{DATA:citrix_adc.log.start_time}" - End_time "%{DATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - (NonHttp_services_accessed %{INT:citrix_adc.log.non_http_services_accessed} - )?Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - (Total_UDP_flows %{INT:citrix_adc.log.total_udp_flows} - )?Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod "%{DATA:citrix_adc.log.logout_method}" - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' - - '^(Logout handler : )?Context %{DATA:citrix_adc.log.username}@%{IP} - SessionId: %{NUMBER:citrix_adc.log.session_id} - User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip (%{IP:citrix_adc.log.nat.ip}|"%{DATA}") - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time "%{DATA:citrix_adc.log.start_time}" - End_time "%{DATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - NonHttp_services_accessed %{INT:citrix_adc.log.non_http_services_accessed} - Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - Total_UDP_flows %{INT:citrix_adc.log.total_udp_flows} - Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod "%{DATA:citrix_adc.log.logout_method}" - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' + - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time "%{DATA:_tmp.start_time}" - End_time "%{DATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - (NonHttp_services_accessed %{INT:citrix_adc.log.non_http_services_accessed} - )?Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - (Total_UDP_flows %{INT:citrix_adc.log.total_udp_flows} - )?Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod "%{DATA:citrix_adc.log.logout_method}" - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' + - '^(Logout handler : )?Context %{DATA:citrix_adc.log.username}@%{IP} - SessionId: %{NUMBER:citrix_adc.log.session_id} - User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip (%{IP:citrix_adc.log.nat.ip}|"%{DATA}") - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Start_time "%{DATA:_tmp.start_time}" - End_time "%{DATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Http_resources_accessed %{INT:citrix_adc.log.http_resources_accessed} - NonHttp_services_accessed %{INT:citrix_adc.log.non_http_services_accessed} - Total_TCP_connections %{INT:citrix_adc.log.total_tcp_connections} - Total_UDP_flows %{INT:citrix_adc.log.total_udp_flows} - Total_policies_allowed %{INT:citrix_adc.log.total_policies_allowed} - Total_policies_denied %{INT:citrix_adc.log.total_policies_denied} - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved}% - LogoutMethod "%{DATA:citrix_adc.log.logout_method}" - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' - grok: tag: grok_sslvpn_icastart if: 'ctx.citrix.name == "ICASTART"' field: citrix.extended.message patterns: - - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - customername( %{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - applicationName %{DATA:citrix_adc.log.application_name} - startTime "%{DATA:citrix_adc.log.start_time}" - connectionId %{WORD:citrix_adc.log.connection_id} ?$' - - '^%{DATA} Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - customername (%{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - applicationName %{DATA:citrix_adc.log.application_name} - startTime "%{DATA:citrix_adc.log.start_time}" - connectionId %{WORD:citrix_adc.log.connection_id} ?$' - + - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - customername( %{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - applicationName %{DATA:citrix_adc.log.application_name} - startTime "%{DATA:_tmp.start_time}" - connectionId %{WORD:citrix_adc.log.connection_id} ?$' + - '^%{DATA} Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - customername (%{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - applicationName %{DATA:citrix_adc.log.application_name} - startTime "%{DATA:_tmp.start_time}" - connectionId %{WORD:citrix_adc.log.connection_id} ?$' + - grok: tag: grok_sslvpn_icaend_connstat if: 'ctx.citrix.name == "ICAEND_CONNSTAT"' field: citrix.extended.message patterns: - - '^%{DATA} ?Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - (SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - )?customername (%{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - startTime "%{DATA:citrix_adc.log.start_time}" - endTime "%{DATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} ? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - connectionId %{WORD:citrix_adc.log.connection_id} ?$' - - '^%{DATA} ?Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - (SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - )?customername (%{WORD:citrix_adc.log.customer_name})? ?- username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - startTime "%{DATA:citrix_adc.log.start_time}" - endTime "%{DATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} ? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - connectionId %{WORD:citrix_adc.log.connection_id} - Total_bytes_wire_send %{INT:citrix_adc.log.total_bytes_wire_send} - Total_bytes_wire_recv %{INT:citrix_adc.log.total_bytes_wire_recieved} ?$' + - '^%{DATA} ?Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - (SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - )?customername (%{WORD:citrix_adc.log.customer_name})? - username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - startTime "%{DATA:_tmp.start_time}" - endTime "%{DATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} ? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - connectionId %{WORD:citrix_adc.log.connection_id} ?$' + - '^%{DATA} ?Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - (SSLRelayAddress %{IP:citrix_adc.log.ssl_relay.address}:%{INT:citrix_adc.log.ssl_relay.port} - )?customername (%{WORD:citrix_adc.log.customer_name})? ?- username:domainname %{DATA:citrix_adc.log.username}:%{DATA:citrix_adc.log.domain_name} - startTime "%{DATA:_tmp.start_time}" - endTime "%{DATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} ? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received} - Total_compressedbytes_send %{INT:citrix_adc.log.total_compressed_bytes_send} - Total_compressedbytes_recv %{INT:citrix_adc.log.total_compressed_bytes_recieved} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - connectionId %{WORD:citrix_adc.log.connection_id} - Total_bytes_wire_send %{INT:citrix_adc.log.total_bytes_wire_send} - Total_bytes_wire_recv %{INT:citrix_adc.log.total_bytes_wire_recieved} ?$' - grok: tag: grok_sslvpn_tcpconnstat if: 'ctx.citrix.name == "TCPCONNSTAT"' field: citrix.extended.message patterns: - - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:citrix_adc.log.start_time}" - End_time "%{GREEDYDATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Total_compressedbytes_send %{NUMBER:citrix_adc.log.total_compressed_bytes_send:int} - Total_compressedbytes_recv %{NUMBER:citrix_adc.log.total_compressed_bytes_recieved:int} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}"$' - - '^Context %{DATA:citrix_adc.log.username}@%{IP} - SessionId: %{NUMBER:citrix_adc.log.session_id} - User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:citrix_adc.log.start_time}" - End_time "%{GREEDYDATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Total_compressedbytes_send %{NUMBER:citrix_adc.log.total_compressed_bytes_send:int} - Total_compressedbytes_recv %{NUMBER:citrix_adc.log.total_compressed_bytes_recieved:int} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' + - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:_tmp.start_time}" - End_time "%{GREEDYDATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Total_compressedbytes_send %{NUMBER:citrix_adc.log.total_compressed_bytes_send:int} - Total_compressedbytes_recv %{NUMBER:citrix_adc.log.total_compressed_bytes_recieved:int} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}"$' + - '^Context %{DATA:citrix_adc.log.username}@%{IP} - SessionId: %{NUMBER:citrix_adc.log.session_id} - User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:_tmp.start_time}" - End_time "%{GREEDYDATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Total_compressedbytes_send %{NUMBER:citrix_adc.log.total_compressed_bytes_send:int} - Total_compressedbytes_recv %{NUMBER:citrix_adc.log.total_compressed_bytes_recieved:int} - Compression_ratio_send %{NUMBER:citrix_adc.log.compression_ratio_send:float}% - Compression_ratio_recv %{NUMBER:citrix_adc.log.compression_ratio_recieved:float}% - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' - grok: tag: grok_sslvpn_tcpconn_timeout @@ -55,15 +55,15 @@ processors: if: 'ctx.citrix.name == "UDPFLOWSTAT"' field: citrix.extended.message patterns: - - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:citrix_adc.log.start_time}" - End_time "%{GREEDYDATA:citrix_adc.log.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' + - '^User %{DATA:citrix_adc.log.user} - Client_ip %{IP:citrix_adc.log.client_ip} - Nat_ip %{IP:citrix_adc.log.nat.ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start_time "%{DATA:_tmp.start_time}" - End_time "%{GREEDYDATA:_tmp.end_time}" - Duration %{DATA:citrix_adc.log.duration} - Total_bytes_send %{NUMBER:citrix_adc.log.total_bytes_send:int} - Total_bytes_recv %{NUMBER:citrix_adc.log.total_bytes_received:int} - Access %{WORD:citrix_adc.log.access} - Group\(s\) "%{DATA:citrix_adc.log.groups}" ?$' - grok: tag: grok_sslvpn_httprequest if: 'ctx.citrix.name == "HTTPREQUEST"' field: citrix.extended.message patterns: - - '^Context (%{USERNAME:citrix_adc.log.username}|%{EMAILADDRESS:citrix_adc.log.username})@%{IP:citrix_adc.log.client_ip} ?- SessionId: %{NUMBER:citrix_adc.log.session_id} ?- %{HOSTNAME:citrix_adc.log.hostname} User (%{USERNAME:citrix_adc.log.user}|%{EMAILADDRESS:citrix_adc.log.user}) ?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - - ?$' - - '^Context (%{USERNAME:citrix_adc.log.username}|%{EMAILADDRESS:citrix_adc.log.username})@%{IP:citrix_adc.log.client_ip} ?- SessionId: %{NUMBER:citrix_adc.log.session_id} ?- %{HOSTNAME:citrix_adc.log.hostname} User (%{USERNAME:citrix_adc.log.user}|%{EMAILADDRESS:citrix_adc.log.user}) ?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:citrix_adc.log.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -?$' + - '^Context (%{USERNAME:citrix_adc.log.username}|%{EMAILADDRESS:citrix_adc.log.username})@%{IP:citrix_adc.log.client_ip} ?- SessionId: %{NUMBER:citrix_adc.log.session_id} ?- %{HOSTNAME:citrix_adc.log.hostname} User (%{USERNAME:citrix_adc.log.user}|%{EMAILADDRESS:citrix_adc.log.user}) ?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:_tmp.timestamp} : SSO is %{WORD:citrix_adc.log.sso_status} : %{WORD:citrix_adc.log.method} %{URIPATHPARAM:citrix_adc.log.request.path} - - ?$' + - '^Context (%{USERNAME:citrix_adc.log.username}|%{EMAILADDRESS:citrix_adc.log.username})@%{IP:citrix_adc.log.client_ip} ?- SessionId: %{NUMBER:citrix_adc.log.session_id} ?- %{HOSTNAME:citrix_adc.log.hostname} User (%{USERNAME:citrix_adc.log.user}|%{EMAILADDRESS:citrix_adc.log.user}) ?: Group\(s\) %{DATA:citrix_adc.log.groups} : Vserver %{IP:citrix_adc.log.vserver.ip}:%{NUMBER:citrix_adc.log.vserver.port} - %{DATA:_tmp.timestamp} %{DATA:citrix_adc.log.timezone} %{WORD:citrix_adc.log.method} %{DATA:citrix_adc.log.request.path} - -?$' - grok: tag: grok_sslvpn_nonhttp_access_denied @@ -91,7 +91,7 @@ processors: if: 'ctx.citrix.name == "CLISEC_CHECK"' field: citrix.extended.message patterns: - - '^%{WORD:citrix_adc.log.alert_type} ?: %{WORD:citrix_adc.log.alert_level} - ClientIP %{IP:citrix_adc.log.client_ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Client_security_expression "%{DATA:citrix_adc.log.client_security_expression}" - ?$' + - '^%{WORD:citrix_adc.log.alert_type} ?: %{WORD:citrix_adc.log.alert_level} - ClientIP %{IP:citrix_adc.log.client_ip} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - Client_security_expression "%{DATA:citrix_adc.log.client_security_expression}" - ?$' - grok: tag: grok_sslvpn_sta_validate_resp @@ -121,21 +121,6 @@ processors: patterns: - '^Logout handler : %{DATA}, for user <%{USERNAME|EMAILADDRESS:citrix_adc.log.username}>$' - - date: - field: citrix_adc.log.timestamp - tag: date_timestamp - target_field: citrix_adc.log.timestamp - formats: - - ISO8601 - - MM/dd/yyyy:HH:mm:ss - - MM/dd/yyyy:HH:mm:ss z - - yyyy/MM/dd:HH:mm:ss - - yyyy/MM/dd:HH:mm:ss z - if: ctx.citrix_adc?.log?.timestamp != null && ctx.citrix_adc.log.timestamp != '' - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: field: citrix_adc.log.client_ip tag: convert_client_ip_to_ip @@ -204,42 +189,6 @@ processors: tag: set_error_message_from_errmsg copy_from: citrix_adc.log.errmsg ignore_empty_value: true - - date: - field: citrix_adc.log.end_time - tag: date_end_time - target_field: citrix_adc.log.end_time - formats: - - yyyy/MM/dd:HH:mm:ss - - MM/dd/yyyy:HH:mm:ss - - ISO8601 - if: ctx.citrix_adc?.log?.end_time != null && ctx.citrix_adc.log.end_time != '' - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - set: - field: event.end - tag: set_event_end_from_end_time - copy_from: citrix_adc.log.end_time - ignore_empty_value: true - - date: - field: citrix_adc.log.start_time - tag: date_start_time - target_field: citrix_adc.log.start_time - formats: - - yyyy/MM/dd:HH:mm:ss - - MM/dd/yyyy:HH:mm:ss - - ISO8601 - if: ctx.citrix_adc?.log?.start_time != null && ctx.citrix_adc.log.start_time != '' - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - set: - field: event.start - tag: set_event_start_from_start_time - copy_from: citrix_adc.log.start_time - ignore_empty_value: true - set: field: group.name tag: set_group_name_from_groups diff --git a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/tcp_and_acl_feature.yml b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/tcp_and_acl_feature.yml index e45b74466f8..f709a4d997b 100644 --- a/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/tcp_and_acl_feature.yml +++ b/packages/citrix_adc/data_stream/log/elasticsearch/ingest_pipeline/tcp_and_acl_feature.yml @@ -6,10 +6,10 @@ processors: tag: grok_tcp_and_acl_feature field: citrix.extended.message patterns: - - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Delink Time %{DATA:citrix_adc.log.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' - - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Delink Time %{DATA:citrix_adc.log.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' - - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start Time %{DATA:citrix_adc.log.start_time}(?: %{DATA:citrix_adc.log.start_time_timezone})? - End Time %{DATA:citrix_adc.log.end_time}(?: %{DATA:citrix_adc.log.end_time_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' - - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.original_destination.ip}:%{INT:citrix_adc.log.original_destination.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.translated_destination.ip}:%{INT:citrix_adc.log.translated_destination.port} - Start Time %{DATA:citrix_adc.log.start_time}(?: %{DATA:citrix_adc.log.start_time_timezone})? - Delink Time %{DATA:citrix_adc.log.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long} - Closure%{SPACE}Reason %{GREEDYDATA:citrix_adc.log.closure_reason}$' + - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Delink Time %{DATA:_tmp.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' + - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Vserver %{IP:citrix_adc.log.vserver.ip}:%{INT:citrix_adc.log.vserver.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Delink Time %{DATA:_tmp.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' + - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.destination.ip}:%{INT:citrix_adc.log.destination.port} - Start Time %{DATA:_tmp.start_time}(?: %{DATA:citrix_adc.log.start_time_timezone})? - End Time %{DATA:_tmp.end_time}(?: %{DATA:citrix_adc.log.end_time_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long}$' + - '^Source %{IP:citrix_adc.log.source.ip}:%{INT:citrix_adc.log.source.port} - Destination %{IP:citrix_adc.log.original_destination.ip}:%{INT:citrix_adc.log.original_destination.port} - NatIP %{IP:citrix_adc.log.nat.ip}:%{INT:citrix_adc.log.nat.port} - Destination %{IP:citrix_adc.log.translated_destination.ip}:%{INT:citrix_adc.log.translated_destination.port} - Start Time %{DATA:_tmp.start_time}(?: %{DATA:citrix_adc.log.start_time_timezone})? - Delink Time %{DATA:_tmp.delink_time}(?: %{DATA:citrix_adc.log.delink_timezone})? - Total_bytes_send %{INT:citrix_adc.log.total_bytes_send:long} - Total_bytes_recv %{INT:citrix_adc.log.total_bytes_received:long} - Closure%{SPACE}Reason %{GREEDYDATA:citrix_adc.log.closure_reason}$' - '^Source %{IP:citrix_adc.log.source.ip} --> Destination %{IP:citrix_adc.log.destination.ip} - Protocol %{WORD:citrix_adc.log.protocol} - Type %{INT:citrix_adc.log.type} - Code %{INT:citrix_adc.log.code} - Time%{SPACE}Stamp %{DATA:citrix_adc.log.timestamp}%{SPACE}\(ms\) - Hitcount %{INT:citrix_adc.log.hit.count:int} - Hit%{SPACE}Rule %{GREEDYDATA:citrix_adc.log.hit.rule} - Action %{WORD:citrix_adc.log.action} - Data$' - '%{GREEDYDATA:citrix_adc.log.message}' ignore_failure: true @@ -134,93 +134,11 @@ processors: tag: set_event_action_from_action copy_from: citrix_adc.log.action ignore_empty_value: true - - script: - description: Convert delink_time via custom format - tag: date_delink_time_custom_format - lang: painless - source: >- - def dateFormat = ctx["_conf"]["custom_date_format"]; - def formatter = DateTimeFormatter.ofPattern(dateFormat); - - ["delink_time", "start_time", "end_time", "timestamp"].forEach(l -> { - - if (ctx.containsKey("citrix_adc")) { - if (ctx.citrix_adc.containsKey("log")) { - if (ctx.citrix_adc.log.containsKey(l)) { - def text = ctx.citrix_adc.log[l]; - def parsedDate = LocalDateTime.parse(text, formatter); - ctx.citrix_adc.log[l] = parsedDate.toString(); - } - } - } - return true; - }); - - if: ctx?._conf?.custom_date_format != null && ctx?._conf?.custom_date_format != '' - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}. Offending format: {{{_conf.custom_date_format}}}.' - - date: - field: citrix_adc.log.delink_time - tag: date_delink_time - target_field: citrix_adc.log.delink_time - formats: - - yyyy/MM/dd:HH:mm:ss - - MM/dd/yyyy:HH:mm:ss - - yyyy/MM/dd:HH:mm:ss - - ISO8601 - if: ctx.citrix_adc?.log?.delink_time != null && ctx.citrix_adc.log.delink_time != '' && ctx?._conf?.custom_date_format == null - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - set: - field: event.end - tag: set_event_end_from_delink_time - copy_from: citrix_adc.log.delink_time - ignore_empty_value: true - - date: - field: citrix_adc.log.end_time - tag: date_end_time - target_field: citrix_adc.log.end_time - formats: - - ISO8601 - - MM/dd/yyyy:HH:mm:ss - - yyyy/MM/dd:HH:mm:ss - if: ctx.citrix_adc?.log?.end_time != null && ctx.citrix_adc.log.end_time != '' && ctx?._conf?.custom_date_format == null - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - set: - field: event.end - tag: set_event_end_from_end_time - copy_from: citrix_adc.log.end_time - ignore_empty_value: true - set: field: event.reason tag: set_event_reason_from_closure_reason copy_from: citrix_adc.log.closure_reason ignore_empty_value: true - - date: - field: citrix_adc.log.start_time - tag: date_start_time - target_field: citrix_adc.log.start_time - formats: - - ISO8601 - - MM/dd/yyyy:HH:mm:ss - - yyyy/MM/dd:HH:mm:ss - if: ctx.citrix_adc?.log?.start_time != null && ctx.citrix_adc.log.start_time != '' && ctx?._conf?.custom_date_format == null - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - set: - field: event.start - tag: set_event_start_from_start_time - copy_from: citrix_adc.log.start_time - ignore_empty_value: true - set: field: network.protocol tag: set_network_protocol_from_protocol diff --git a/packages/citrix_adc/manifest.yml b/packages/citrix_adc/manifest.yml index cfccc77bb70..e18933f501f 100644 --- a/packages/citrix_adc/manifest.yml +++ b/packages/citrix_adc/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: citrix_adc title: Citrix ADC -version: "1.10.0" +version: "1.11.0" description: This Elastic integration collects logs and metrics from Citrix ADC product. type: integration categories: