Change Windows fluentbit configuration (#45)

* Change Windows fluentbit configuration 1. Move kubelet and kubelet events from Windows to dataplane log group * Refactor resource count to variables in validator package
aws-observability · Jun 2, 2024 · 45b8581 · 45b8581
1 parent 04d2509
commit 45b8581
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 6 deletions.
diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml
@@ -331,11 +331,22 @@ containerLogs:
             Rotate_Wait         30
             Refresh_Interval    10
             Read_from_Head      ${READ_FROM_HEAD}
+
+          [INPUT]
+            Name                winlog
+            Channels            EKS
+            DB                  C:\\var\\fluent-bit\\state\\flb_eks_winlog.db
+            Interval_Sec        60
 
           [FILTER]
             Name                aws
             Match               dataplane.*
             imds_version        v2
+
+          [FILTER]
+            Name                aws
+            Match               winlog.*
+            imds_version        v2
 
           [OUTPUT]
             Name                cloudwatch_logs
@@ -345,10 +356,19 @@ containerLogs:
             log_stream_prefix   ${HOST_NAME}-
             auto_create_group   true
             extra_user_agent    container-insights
+
+          [OUTPUT]
+            Name                cloudwatch_logs
+            Match               winlog.*
+            region              ${AWS_REGION}
+            log_group_name      /aws/containerinsights/${CLUSTER_NAME}/dataplane
+            log_stream_name     ${HOST_NAME}.windows.kubelet.kubeproxy.service
+            auto_create_group   true
+            extra_user_agent    container-insights
         host-log.conf: |
           [INPUT]
             Name                winlog
-            Channels            EKS, System
+            Channels            System
             DB                  C:\\var\\fluent-bit\\state\\flb_system_winlog.db
             Interval_Sec        60
       
@@ -362,7 +382,7 @@ containerLogs:
             Match               winlog.*
             region              ${AWS_REGION}
             log_group_name      /aws/containerinsights/${CLUSTER_NAME}/host
-            log_stream_prefix   ${HOST_NAME}.
+            log_stream_name     ${HOST_NAME}.windows.system.events
             auto_create_group   true
             extra_user_agent    container-insights
 

diff --git a/integration-tests/amazon-cloudwatch-observability/validator/resourceCount_linuxonly_test.go b/integration-tests/amazon-cloudwatch-observability/validator/resourceCount_linuxonly_test.go
@@ -7,6 +7,14 @@
 package validator
 
 const (
+	// Services count for CW agent on Linux and Windows
+	serviceCountLinux   = 6
+	serviceCountWindows = 3
+
+	// DaemonSet count for CW agent on Linux and Windows
+	daemonsetCountLinux   = 4
+	daemonsetCountWindows = 2
+
 	// Pods count for CW agent on Linux and Windows
 	podCountLinux   = 3
 	podCountWindows = 0

diff --git a/...ration-tests/amazon-cloudwatch-observability/validator/resourceCount_windowslinux_test.go b/...ration-tests/amazon-cloudwatch-observability/validator/resourceCount_windowslinux_test.go
@@ -7,6 +7,14 @@
 package validator
 
 const (
+	// Services count for CW agent on Linux and Windows
+	serviceCountLinux   = 6
+	serviceCountWindows = 3
+
+	// DaemonSet count for CW agent on Linux and Windows
+	daemonsetCountLinux   = 4
+	daemonsetCountWindows = 2
+
 	// Pods count for CW agent on Linux and Windows
 	podCountLinux   = 3
 	podCountWindows = 2

diff --git a/integration-tests/amazon-cloudwatch-observability/validator/validateResources_test.go b/integration-tests/amazon-cloudwatch-observability/validator/validateResources_test.go
@@ -40,7 +40,10 @@ const (
 )
 
 const (
-	podCount = podCountLinux + podCountWindows
+	deploymentCount = 1
+	podCount        = podCountLinux + podCountWindows
+	serviceCount    = serviceCountLinux + serviceCountWindows
+	daemonsetCount  = daemonsetCountLinux + daemonsetCountWindows
 )
 
 func TestOperatorOnEKs(t *testing.T) {
@@ -88,7 +91,7 @@ func TestOperatorOnEKs(t *testing.T) {
 	//Validating the services
 	services, err := ListServices(nameSpace, clientSet)
 	assert.NoError(t, err)
-	assert.Len(t, services.Items, 9)
+	assert.Len(t, services.Items, serviceCount)
 	for _, service := range services.Items {
 		fmt.Println("service name: " + service.Name + " namespace:" + service.Namespace)
 		// matches
@@ -112,7 +115,7 @@ func TestOperatorOnEKs(t *testing.T) {
 	for _, deployment := range deployments.Items {
 		fmt.Println("deployment name: " + deployment.Name + " namespace:" + deployment.Namespace)
 	}
-	assert.Len(t, deployments.Items, 1)
+	assert.Len(t, deployments.Items, deploymentCount)
 	// matches
 	// - amazon-cloudwatch-observability-controller-manager
 	assert.Equal(t, addOnName+"-controller-manager", deployments.Items[0].Name)
@@ -124,7 +127,7 @@ func TestOperatorOnEKs(t *testing.T) {
 	//Validating the Daemon Sets
 	daemonSets, err := ListDaemonSets(nameSpace, clientSet)
 	assert.NoError(t, err)
-	assert.Len(t, daemonSets.Items, 6)
+	assert.Len(t, daemonSets.Items, daemonsetCount)
 	for _, daemonSet := range daemonSets.Items {
 		fmt.Println("daemonSet name: " + daemonSet.Name + " namespace:" + daemonSet.Namespace)
 		// matches