Skip to content

Commit

Permalink
Merge pull request #163 from logzio/feature/otel-logging-kubernetes
Browse files Browse the repository at this point in the history
Use `logzio-logs-collector` for default k8s logging
  • Loading branch information
yotamloe authored May 27, 2024
2 parents e0b2e5d + 57aa3d9 commit 1caf6f5
Show file tree
Hide file tree
Showing 39 changed files with 439 additions and 127 deletions.
8 changes: 4 additions & 4 deletions config-templates/helm.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,18 @@
"targetElement": "datasources[?(@.name=='Kubernetes')].telemetries[?(@.type=='LOG_ANALYTICS')]",
"conditionalTemplates": [
{
"template": " --set logzio-fluentd.secrets.logzioShippingToken='<<LOGZIO_LOGS_SHIPPING_TOKEN>>' \\"
"template": " --set logzio-logs-collector.secrets.logzioLogsToken='<<LOGZIO_LOGS_SHIPPING_TOKEN>>' \\"
},
{
"template": " --set logzio-fluentd.secrets.logzioListener='<<LOGZIO_LISTENER_ADDRESS>>' \\"
"template": " --set logzio-logs-collector.secrets.logzioRegion='<<LOGZIO_ACCOUNT_REGION_CODE>>' \\"
},
{
"template": " --set logzio-fluentd.fargateLogRouter.enabled=true \\",
"template": " --set logzio-logs-collector.fargateLogRouter.enabled=true \\",
"condition": "$[?(@ == 'EKS')]",
"targetElement": "datasources[?(@.name=='Kubernetes')].params[?(@.name=='isFargate')].value"
},
{
"template": " --set logzio-fluentd.env_id='{{value}}' \\",
"template": " --set logzio-logs-collector.secrets.env_id='{{value}}' \\",
"targetElement": "datasources[?(@.name=='Kubernetes')].params[?(@.name=='envID')].value",
"keepAsPlaceholder": true
}
Expand Down
2 changes: 1 addition & 1 deletion configs/2-1-k8s-eks.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"name": "Kubernetes",
"logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
"label": "Kubernetes",
"description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
"description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
"hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
"params": [
{
Expand Down
2 changes: 1 addition & 1 deletion configs/2-2-k8s-aks.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"name": "Kubernetes",
"logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
"label": "Kubernetes",
"description": "Logz.io Uses OpenTelemetry and Fluentd with Helm charts to deploy and monitor Kubernetes architecture",
"description": "Logz.io Uses OpenTelemetry with Helm charts to deploy and monitor Kubernetes architecture",
"params": [
{
"type": "text",
Expand Down
2 changes: 1 addition & 1 deletion configs/2-3-k8s-gke.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"name": "Kubernetes",
"logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
"label": "Kubernetes",
"description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
"description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
"hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
"params": [
{
Expand Down
2 changes: 1 addition & 1 deletion configs/2-4-k8s-digitalocean.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"name": "Kubernetes",
"logo": "https://dytvr9ot2sszz.cloudfront.net/logz-docs/shipper-logos/kubernetes.svg",
"label": "Kubernetes",
"description": "Logz.io uses OpenTelemetry instrumentation with Fluentd and Helm charts to deploy and monitor Kubernetes architecture.",
"description": "Logz.io uses OpenTelemetry instrumentation and collector and Helm charts to deploy and monitor Kubernetes architecture.",
"hint": "Select the type of data you'd like to ship: logs, metrics, and traces.",
"params": [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function build_enable_fargate_helm_set {
send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
write_log "$LOG_LEVEL_DEBUG" "$message"

local helm_set=" --set logzio-fluentd.fargateLogRouter.enabled=true"
local helm_set=" --set logzio-logs-collector.fargateLogRouter.enabled=true"

message="Enable Fargate Helm set is '$helm_set'"
send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function build_enable_fargate_helm_set {
send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
write_log "$LOG_LEVEL_DEBUG" "$message"

local helm_set=" --set logzio-fluentd.fargateLogRouter.enabled=true"
local helm_set=" --set logzio-logs-collector.fargateLogRouter.enabled=true"

message="Enable Fargate Helm set is '$helm_set'"
send_log_to_logzio "$LOG_LEVEL_DEBUG" "$message" "$LOG_STEP_LOGS" "$LOG_SCRIPT_LOGS" "$func_name" "$AGENT_ID" "$PLATFORM" "$SUB_TYPE" "$CURRENT_DATA_SOURCE"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,12 @@ load_logs_utils
execute_task 'get_logzio_logs_token' 'Getting Logz.io logs token'
# Build enable logs Helm set
execute_task 'build_enable_logs_helm_set' 'Building enable logs Helm set'
# Build Logz.io logs listener url Helm set
execute_task 'build_logzio_logs_listener_url_helm_set' 'Building Logz.io logs listener url Helm set'
# Build enable opentelemetry log collection Helm set
execute_task 'build_enable_opentelemetry_log_collection_helm_set' 'Building enable opentelemetry log collection Helm set'
# Build disable fluentd log collection Helm set
execute_task 'build_disable_fluentd_log_collection_helm_set' 'Building disable fluentd log collection Helm set'
# Build Logz.io logs region Helm set
execute_task 'build_logzio_logs_region_helm_set' 'Building Logz.io logs region Helm set'
# Build Logz.io logs token Helm set
execute_task 'build_logzio_logs_token_helm_set' 'Building Logz.io logs token Helm set'
# Build environment id Helm set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -385,8 +385,8 @@ function Build-TolerationsHelmSets {
$Operator = 'Equal'

if ($script:IsLogsOptionSelected) {
$TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].value=$Value"
$TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].value=$Value"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].value=$Value"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].value=$Value"
}
if ($script:IsMetricsOptionSelected -or $script:IsTracesOptionSelected) {
$TolerationHelmSets += " --set-string logzio-k8s-telemetry.prometheus-pushgateway.tolerations[$TolerationIndex].value=$Value"
Expand All @@ -397,12 +397,12 @@ function Build-TolerationsHelmSets {
}

if ($script:IsLogsOptionSelected) {
$TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].key=$Key"
$TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].operator=$Operator"
$TolerationHelmSets += " --set-string logzio-fluentd.daemonset.tolerations[$TolerationIndex].effect=$Effect"
$TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].key=$Key"
$TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].operator=$Operator"
$TolerationHelmSets += " --set-string logzio-fluentd.windowsDaemonset.tolerations[$TolerationIndex].effect=$Effect"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].key=$Key"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].operator=$Operator"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].effect=$Effect"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].key=$Key"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].operator=$Operator"
$TolerationHelmSets += " --set-string logzio-logs-collector.tolerations[$TolerationIndex].effect=$Effect"
}
if ($script:IsMetricsOptionSelected -or $script:IsTracesOptionSelected) {
$TolerationHelmSets += " --set-string logzio-k8s-telemetry.prometheus-pushgateway.tolerations[$TolerationIndex].key=$Key"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,21 +56,59 @@ function Build-EnableLogsHelmSet {
Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
}

# Builds Logz.io logs listener url Helm set
# Builds enable logzio-logs-collector Helm set
# Input:
# ---
# Output:
# LogHelmSets - Containt all the Helm sets for logging
# helmSets - Contains all the Helm sets
function Build-EnableOtelLogCollectionHelmSet {
$local:FuncName = $MyInvocation.MyCommand.Name

$local:Message = 'Building enable opentelemetry log collection Helm set ...'
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Write-Log $script:LogLevelDebug $Message

$local:HelmSet = " --set logzio-logs-collector.enabled=true"

$Message = "Enable opentelemetry log collection Helm set is '$HelmSet'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Write-Log $script:LogLevelDebug $Message

Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
}

# Builds disable logzio-fluentd Helm set
# Input:
# ---
# Output:
# LogHelmSets - Containt all the Helm sets for logging
# helmSets - Contains all the Helm sets
function Build-DisableFluentdHelmSet {
$local:FuncName = $MyInvocation.MyCommand.Name

$local:HelmSet = " --set logzio-fluentd.enabled=false"

Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
Write-TaskPostRun "`$script:HelmSets += '$HelmSet'"
}

# Builds Logz.io logs region Helm set
# Input:
# FuncArgs - Hashtable {ListenerUrl = $script:ListenerUrl}
# Output:
# LogHelmSets - Containt all the Helm sets for logging
# HelmSets - Contains all the Helm sets
function Build-LogzioLogsListenerUrlHelmSet {
function Build-LogzioLogsRegionHelmSet {
param (
[hashtable]$FuncArgs
)

$local:ExitCode = 2
$local:FuncName = $MyInvocation.MyCommand.Name

$local:Message = 'Building Logz.io logs listener URL Helm set ...'
$local:Message = 'Building Logz.io logs region Helm set ...'
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Write-Log $script:LogLevelDebug $Message

Expand All @@ -85,10 +123,16 @@ function Build-LogzioLogsListenerUrlHelmSet {

$local:ListenerUrl = $FuncArgs.ListenerUrl

$local:HelmSet = " --set logzio-fluentd.secrets.logzioListener=$ListenerUrl"
$local:Region = Get-LogzioRegion $ListenerUrl

$Message = "Logz.io logs listener url Helm set is '$HelmSet'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
$Message = "Logz.io region is '$LogzioRegion'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepTraces $script:LogScriptTraces $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Write-Log $script:LogLevelDebug $Message
# changed from fluentd
$local:HelmSet = " --set logzio-logs-collector.secrets.logzioRegion=$LogzioRegion"

$local:Message = "Logz.io region Helm set is '$HelmSet'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepTraces $script:LogScriptTraces $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Write-Log $script:LogLevelDebug $Message

Write-TaskPostRun "`$script:LogHelmSets += '$HelmSet'"
Expand Down Expand Up @@ -124,7 +168,7 @@ function Build-LogzioLogsTokenHelmSet {

$local:LogsToken = $FuncArgs.LogsToken

$local:HelmSet = " --set logzio-fluentd.secrets.logzioShippingToken=$LogsToken"
$local:HelmSet = " --set logzio-logs-collector.secrets.logzioLogsToken=$LogsToken"

$Message = "Logz.io logs token Helm set is '$HelmSet'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Expand Down Expand Up @@ -171,7 +215,7 @@ function Build-EnvironmentIdHelmSet {
return
}

$local:HelmSet = " --set logzio-fluentd.env_id=$EnvId"
$local:HelmSet = " --set logzio-logs-collector.env_id=$EnvId"

$Message = "Environment id Helm set is '$HelmSet'"
Send-LogToLogzio $script:LogLevelDebug $Message $script:LogStepLogs $script:LogScriptLogs $FuncName $script:AgentId $script:Platform $script:Subtype $script:CurrentDataSource
Expand Down
Loading

0 comments on commit 1caf6f5

Please sign in to comment.