Additional changes

diagnostics/opensearch/filebeat/conf/filebeat-k8s.yml

@@ -6,9 +6,52 @@ filebeat.autodiscover:
         type: container
         paths:
           - /var/log/containers/*-${data.container.id}.log  # CRI path
+        ignore_older: 1d
+processors:
+#Drop events from other namespaces except Duplo tenants.
+- drop_event:
+    when.not.regexp:
+      kubernetes.namespace: "^duploservices-*"
+- add_cloud_metadata: ~
+#Currently TENANT_NAME is not passed as ENV variable. Using labels to extract TENANT_NAME            
+- copy_fields:
+    fields:
+      - from: "kubernetes.labels.tenantname"
+        to: "tenant.name"
+      - from: "kubernetes.labels.tenantid"
+        to: "tenant.id"  
+    fail_on_error: false
+    ignore_missing: true    
+#strip out duploservice- and actual TENANT_NAME 
+- replace:
+    fields:
+      - field: "tenant.name"
+        pattern: "duploservices-"
+        replacement: ""
+    ignore_missing: true
+    fail_on_error: false
+#Add fields to support tenant level index    
+- add_fields:
+    target: ''
+    fields:
+      tenantLevelIndex: '${TENANT_LEVEL_INDEX:unknown}'
+- add_fields:
+    target: ''
+    fields:
+      datastream: '${ALIAS:unknown}'         
 setup.template.name: "filebeat-%{[agent.version]}"
 setup.template.pattern: "filebeat-%{[agent.version]}-*"    
 output.elasticsearch:
   hosts: '${ELASTIC_HOST}'
+  indices:
+    - index: "%{[datastream]}"
+      when.not.equals:
+        datastream: "unknown"  
+    - index: "filebeat-%{[agent.version]}-%{[tenant.name]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
+      when.equals:
+        serviceLevelIndex: "yes"
+    - index: "filebeat-%{[agent.version]}-%{[tenant.name]}-%{+yyyy.MM.dd}"
+      when.equals:
+        tenantLevelIndex: "yes"
   bulk_max_size: '${BULK_MAX_SIZE:50}'
-  worker: '${WORKER_COUNT:1}'
+  worker: '${WORKER_COUNT:1}'

diagnostics/opensearch/filebeat/conf/filebeat.yml

@@ -2,6 +2,11 @@ filebeat.autodiscover:
   providers:
     - type: docker
       hints.enabled: true
+      hints.default_config:
+        type: container
+        paths:
+          - /var/lib/docker/containers/${data.docker.container.id}/*.log  # CRI path
+        ignore_older: 1d  
 processors:
 - add_cloud_metadata: ~
 - add_fields: