From 9de0a9cc28d081bc7f4797f7eee90559caa9cd88 Mon Sep 17 00:00:00 2001
From: Srikar Naramsetty <srikar@duplocloud.net>
Date: Mon, 4 Sep 2023 20:06:06 +0530
Subject: [PATCH] Additional changes

---
 .../opensearch/filebeat/conf/filebeat-k8s.yml | 45 ++++++++++++++++++-
 .../opensearch/filebeat/conf/filebeat.yml     |  5 +++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/diagnostics/opensearch/filebeat/conf/filebeat-k8s.yml b/diagnostics/opensearch/filebeat/conf/filebeat-k8s.yml
index 1c4c57b..ddf4cac 100644
--- a/diagnostics/opensearch/filebeat/conf/filebeat-k8s.yml
+++ b/diagnostics/opensearch/filebeat/conf/filebeat-k8s.yml
@@ -6,9 +6,52 @@ filebeat.autodiscover:
         type: container
         paths:
           - /var/log/containers/*-${data.container.id}.log  # CRI path
+        ignore_older: 1d
+processors:
+#Drop events from other namespaces except Duplo tenants.
+- drop_event:
+    when.not.regexp:
+      kubernetes.namespace: "^duploservices-*"
+- add_cloud_metadata: ~
+#Currently TENANT_NAME is not passed as ENV variable. Using labels to extract TENANT_NAME            
+- copy_fields:
+    fields:
+      - from: "kubernetes.labels.tenantname"
+        to: "tenant.name"
+      - from: "kubernetes.labels.tenantid"
+        to: "tenant.id"  
+    fail_on_error: false
+    ignore_missing: true    
+#strip out duploservice- and actual TENANT_NAME 
+- replace:
+    fields:
+      - field: "tenant.name"
+        pattern: "duploservices-"
+        replacement: ""
+    ignore_missing: true
+    fail_on_error: false
+#Add fields to support tenant level index    
+- add_fields:
+    target: ''
+    fields:
+      tenantLevelIndex: '${TENANT_LEVEL_INDEX:unknown}'
+- add_fields:
+    target: ''
+    fields:
+      datastream: '${ALIAS:unknown}'         
 setup.template.name: "filebeat-%{[agent.version]}"
 setup.template.pattern: "filebeat-%{[agent.version]}-*"    
 output.elasticsearch:
   hosts: '${ELASTIC_HOST}'
+  indices:
+    - index: "%{[datastream]}"
+      when.not.equals:
+        datastream: "unknown"  
+    - index: "filebeat-%{[agent.version]}-%{[tenant.name]}-%{[kubernetes.container.name]}-%{+yyyy.MM.dd}"
+      when.equals:
+        serviceLevelIndex: "yes"
+    - index: "filebeat-%{[agent.version]}-%{[tenant.name]}-%{+yyyy.MM.dd}"
+      when.equals:
+        tenantLevelIndex: "yes"
   bulk_max_size: '${BULK_MAX_SIZE:50}'
-  worker: '${WORKER_COUNT:1}'
+  worker: '${WORKER_COUNT:1}'
\ No newline at end of file
diff --git a/diagnostics/opensearch/filebeat/conf/filebeat.yml b/diagnostics/opensearch/filebeat/conf/filebeat.yml
index fef6fdb..40a6ad3 100644
--- a/diagnostics/opensearch/filebeat/conf/filebeat.yml
+++ b/diagnostics/opensearch/filebeat/conf/filebeat.yml
@@ -2,6 +2,11 @@ filebeat.autodiscover:
   providers:
     - type: docker
       hints.enabled: true
+      hints.default_config:
+        type: container
+        paths:
+          - /var/lib/docker/containers/${data.docker.container.id}/*.log  # CRI path
+        ignore_older: 1d  
 processors:
 - add_cloud_metadata: ~
 - add_fields: