Use logzio-logs-collector for default k8s logging

config-templates/helm.json

@@ -5,18 +5,24 @@
     "targetElement": "datasources[?(@.name=='Kubernetes')].telemetries[?(@.type=='LOG_ANALYTICS')]",
     "conditionalTemplates": [
       {
-        "template": "  --set logzio-fluentd.secrets.logzioShippingToken='<<LOGZIO_LOGS_SHIPPING_TOKEN>>' \\"
+        "template": "  --set logzio-logs-collector.enabled=true \\"
       },
       {
-        "template": "  --set logzio-fluentd.secrets.logzioListener='<<LOGZIO_LISTENER_ADDRESS>>' \\"
+        "template": "  --set logzio-fluentd.enabled=false \\"
       },
       {
-        "template": "  --set logzio-fluentd.fargateLogRouter.enabled=true \\",
+        "template": "  --set logzio-logs-collector.secrets.logzioLogsToken='<<LOGZIO_LOGS_SHIPPING_TOKEN>>' \\"
+      },
+      {
+        "template": "  --set logzio-logs-collector.secrets.logzioRegion='<<LOGZIO_ACCOUNT_REGION_CODE>>' \\"
+      },
+      {
+        "template": "  --set logzio-logs-collector.fargateLogRouter.enabled=true \\",
         "condition": "$[?(@ == 'EKS')]",
         "targetElement": "datasources[?(@.name=='Kubernetes')].params[?(@.name=='isFargate')].value"
       },
       {
-        "template": "  --set logzio-fluentd.env_id='{{value}}' \\",
+        "template": "  --set logzio-logs-collector.secrets.env_id='{{value}}' \\",
         "targetElement": "datasources[?(@.name=='Kubernetes')].params[?(@.name=='envID')].value",
         "keepAsPlaceholder": true
       }

configs/2-1-k8s-eks.json

@@ -21,7 +21,7 @@
           "name": "Kubernetes",
           "logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
           "label": "Kubernetes",
-          "description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
+          "description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
           "hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
           "params": [
             {

configs/2-2-k8s-aks.json

@@ -19,7 +19,7 @@
           "name": "Kubernetes",
           "logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
           "label": "Kubernetes",
-          "description": "Logz.io Uses OpenTelemetry and Fluentd with Helm charts to deploy and monitor Kubernetes architecture",
+          "description": "Logz.io Uses OpenTelemetry with Helm charts to deploy and monitor Kubernetes architecture",
           "params": [
             {
               "type": "text",

configs/2-3-k8s-gke.json

@@ -21,7 +21,7 @@
           "name": "Kubernetes",
           "logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
           "label": "Kubernetes",
-          "description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
+          "description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
           "hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
           "params": [
             {

configs/2-4-k8s-digitalocean.json

@@ -21,7 +21,7 @@
           "name": "Kubernetes",
           "logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
           "label": "Kubernetes",
-          "description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
+          "description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
           "hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
           "params": [
             {

datasources/linux/kubernetes/aks/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/linux/kubernetes/digitalocean/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/linux/kubernetes/eks/kubernetes/telemetry/logs/functions.bash

@@ -54,7 +54,7 @@ function build_enable_fargate_helm_set {
     send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
     write_log "$LOG_LEVEL_DEBUG" "$message"
 
-    local helm_set=" --set logzio-fluentd.fargateLogRouter.enabled=true"
+    local helm_set=" --set logzio-logs-collector.fargateLogRouter.enabled=true"
 
     message="Enable Fargate Helm set is '$helm_set'"
     send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"

datasources/linux/kubernetes/eks/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/linux/kubernetes/gke/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/mac/kubernetes/aks/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/mac/kubernetes/digitalocean/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/mac/kubernetes/eks/kubernetes/telemetry/logs/functions.bash

@@ -54,7 +54,7 @@ function build_enable_fargate_helm_set {
     send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
     write_log "$LOG_LEVEL_DEBUG" "$message"
 
-    local helm_set=" --set logzio-fluentd.fargateLogRouter.enabled=true"
+    local helm_set=" --set logzio-logs-collector.fargateLogRouter.enabled=true"
 
     message="Enable Fargate Helm set is '$helm_set'"
     send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"

datasources/mac/kubernetes/eks/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/mac/kubernetes/gke/kubernetes/telemetry/logs/logs.bash

@@ -19,8 +19,12 @@ load_logs_utils
 execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
 # Build enable logs Helm set
 execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
-# Build Logz.io logs listener url Helm set
-execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
+# Build enable opentelemetry log collection Helm set
+execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
+# Build disable fluentd log collection Helm set
+execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
+# Build Logz.io logs region Helm set
+execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
 # Build Logz.io logs token Helm set
 execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
 # Build environment id Helm set

datasources/windows/kubernetes/aks/kubernetes/installer/functions.ps1

@@ -385,8 +385,8 @@ function Build-TolerationsHelmSets {
             $Operator = 'Equal'
 
             if ($script:IsLogsOptionSelected) {
-                $TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].value=$Value"
-                $TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].value=$Value"
+                $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].value=$Value"
+                $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].value=$Value"
             }
             if ($script:IsMetricsOptionSelected -or $script:IsTracesOptionSelected) {
                 $TolerationHelmSets += " --set-string logzio-k8s-telemetry.prometheus-pushgateway.tolerations[$TolerationIndex].value=$Value"
@@ -397,12 +397,12 @@ function Build-TolerationsHelmSets {
         }
 
         if ($script:IsLogsOptionSelected) {
-            $TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].key=$Key"
-            $TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].operator=$Operator"
-            $TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].effect=$Effect"
-            $TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].key=$Key"
-            $TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].operator=$Operator"
-            $TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].effect=$Effect"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].key=$Key"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].operator=$Operator"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].effect=$Effect"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].key=$Key"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].operator=$Operator"
+            $TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].effect=$Effect"
         }
         if ($script:IsMetricsOptionSelected -or $script:IsTracesOptionSelected) {
             $TolerationHelmSets += " --set-string logzio-k8s-telemetry.prometheus-pushgateway.tolerations[$TolerationIndex].key=$Key"

datasources/windows/kubernetes/aks/kubernetes/telemetry/logs/functions.ps1

@@ -56,21 +56,59 @@ function Build-EnableLogsHelmSet {
     Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
 }
 
-# Builds Logz.io logs listener url Helm set
+# Builds enable logzio-logs-collector Helm set
+# Input:
+#   ---
+# Output:
+#   LogHelmSets - Containt all the Helm sets for logging
+#   helmSets - Contains all the Helm sets
+function Build-EnableOtelLogCollectionHelmSet {
+    $local:FuncName = $MyInvocation.MyCommand.Name
+
+    $local:Message = 'Building enable opentelemetry log collection Helm set ...'
+    Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
+    Write-Log $script:LogLevelDebug $Message
+
+    $local:HelmSet = " --set logzio-logs-collector.enabled=true"
+
+    $Message = "Enable opentelemetry log collection Helm set is '$HelmSet'"
+    Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
+    Write-Log $script:LogLevelDebug $Message
+
+    Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
+    Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
+}
+
+# Builds disable logzio-fluentd Helm set
+# Input:
+#   ---
+# Output:
+#   LogHelmSets - Containt all the Helm sets for logging
+#   helmSets - Contains all the Helm sets
+function Build-DisableFluentdHelmSet {
+    $local:FuncName = $MyInvocation.MyCommand.Name
+
+    $local:HelmSet = " --set logzio-fluentd.enabled=false"
+
+    Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
+    Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
+}
+
+# Builds Logz.io logs region Helm set
 # Input:
 #   FuncArgs - Hashtable {ListenerUrl = $script:ListenerUrl}
 # Output:
 #   LogHelmSets - Containt all the Helm sets for logging
 #   HelmSets - Contains all the Helm sets
-function Build-LogzioLogsListenerUrlHelmSet {
+function Build-LogzioLogsRegionHelmSet {
     param (
         [hashtable]$FuncArgs
     )
 
     $local:ExitCode = 2
     $local:FuncName = $MyInvocation.MyCommand.Name
 
-    $local:Message = 'Building Logz.io logs listener URL Helm set ...'
+    $local:Message = 'Building Logz.io logs region Helm set ...'
     Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
     Write-Log $script:LogLevelDebug $Message
 
@@ -85,10 +123,16 @@ function Build-LogzioLogsListenerUrlHelmSet {
 
     $local:ListenerUrl = $FuncArgs.ListenerUrl
 
-    $local:HelmSet = " --set logzio-fluentd.secrets.logzioListener=$ListenerUrl"
+    $local:Region = Get-LogzioRegion $ListenerUrl
 
-    $Message = "Logz.io logs listener url Helm set is '$HelmSet'"
-    Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
+    $Message = "Logz.io region is '$LogzioRegion'"
+    Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepTraces $script:LogScriptTraces $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
+    Write-Log $script:LogLevelDebug $Message
+    # changed from fluentd
+    $local:HelmSet = " --set logzio-logs-collector.secrets.logzioRegion=$LogzioRegion"
+
+    $local:Message = "Logz.io region Helm set is '$HelmSet'"
+    Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepTraces $script:LogScriptTraces $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
     Write-Log $script:LogLevelDebug $Message
 
     Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
@@ -124,7 +168,7 @@ function Build-LogzioLogsTokenHelmSet {
 
     $local:LogsToken = $FuncArgs.LogsToken
 
-    $local:HelmSet = " --set logzio-fluentd.secrets.logzioShippingToken=$LogsToken"
+    $local:HelmSet = " --set logzio-logs-collector.secrets.logzioLogsToken=$LogsToken"
 
     $Message = "Logz.io logs token Helm set is '$HelmSet'"
     Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
@@ -171,7 +215,7 @@ function Build-EnvironmentIdHelmSet {
         return
     }
 
-    $local:HelmSet = " --set logzio-fluentd.env_id=$EnvId"
+    $local:HelmSet = " --set logzio-logs-collector.env_id=$EnvId"
 
     $Message = "Environment id Helm set is '$HelmSet'"
     Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource