Merge pull request #7 from orange-cloudfoundry/use-RELP

Introduce RELP in RSyslog forwarding

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: build-and-release
 
 on:
   push:
-    # not not consider simplec commit
+    # do not consider simples commits
     branches:
       - '!*'
     # consider only release and pre-release tags

README.md

@@ -79,7 +79,7 @@ The job also configures local `logrotate` in order to rotate and compress logs e
 Rotated logs are stored in the same directories with the `-%Y%m%d%H.gz` suffix.
 
 The number of kept rotations can be configured `loghost_concentrator.logrotate.max-hours` property
-with a default value of `360` (ie: 15 days).
+with a default value of `360` (i.e.: 15 days).
 
 #### Forwarding and clustering
 
@@ -101,7 +101,7 @@ target objects as follows:
   targets:
     - address: hostname
       port: port
-      transport: tcp|udp
+      transport: tcp|udp|relp
     - ...
 ```
 
@@ -175,7 +175,7 @@ to give the list of logs files that the exported should watch.
 > at exporter startup. Because rsyslog files are created on the fly when events are received, the
 > job creates required directories in its `pre-start` script.
 
-In addition to user defined metrics, the exporter provides
+In addition to user-defined metrics, the exporter provides
 [builtin metrics][grok-builtin-metrics].
 
 Ops-files provided in the release also provide metrics, as described in the [usage section](#usage).

jobs/loghost_concentrator/spec

@@ -13,6 +13,13 @@ templates:
 packages: []
 
 properties:
+  loghost_concentrator.syslog.relp.enabled:
+    description: "Enable RELP module"
+    default: false
+  loghost_concentrator.syslog.relp.port:
+    description: "Port for RELP module"
+    default: 2514
+
   loghost_concentrator.syslog.tcp.enabled:
     description: "Enable TCP module"
     default: false
@@ -39,7 +46,7 @@ properties:
       Hash that describe log forwarding. Each key defines a target type composed by
       - a list of conditions that must match to trigger the forward
       - a list of remote endpoints to forward the log to
-      Conditions must be writter in RainerScript. Available fields are standard rsyslog
+      Conditions must be written in RainerScript. Available fields are standard rsyslog
       variables (like $msg, $programname, etc) and parsed structured-data defined in
       instance@47450:
       - $.director
@@ -66,6 +73,9 @@ properties:
             - address: 10.120.53
               port: 514
               transport: udp
+            - address: 198.51.100.0
+              port: 2514
+              transport: relp
 
   loghost_concentrator.syslog.longterm-jobs:
     description: "List of specific jobs that must be kept for long period of time"

jobs/loghost_concentrator/templates/rsyslog-loghost.conf.erb

@@ -1,32 +1,48 @@
 $PrivDropToUser root
 $PrivDropToGroup root
 
-<% if_p('loghost_concentrator.syslog.tls.cert') do |v| %>
+<% if_p('loghost_concentrator.syslog.tls.cert') do -%>
 global(
 DefaultNetstreamDriver="gtls"
 DefaultNetstreamDriverCAFile="/var/vcap/jobs/loghost_concentrator/config/ssl_ca.cert"
 DefaultNetstreamDriverCertFile="/var/vcap/jobs/loghost_concentrator/config/ssl.cert"
 DefaultNetstreamDriverKeyFile="/var/vcap/jobs/loghost_concentrator/config/ssl.key"
 )
-<% end %>
+<% end -%>
 
-
-<% if p('loghost_concentrator.syslog.udp.enabled') %>
+<% if p('loghost_concentrator.syslog.relp.enabled') -%>
+<%# module RELP is already loaded, with TLS support, in /etc/rsyslog.conf %>
+input(type="imrelp" port="<%= p('loghost_concentrator.syslog.relp.port') %>"
+  maxDataSize="10k"
+  keepAlive="on"
+  keepAlive.Probes="6"
+  keepAlive.Interval="5"
+  keepAlive.Time="5"
+  <% if_p('loghost_concentrator.syslog.tls.cert') do -%>
+  tls="on"
+  tls.cacert="/var/vcap/jobs/loghost_concentrator/config/ssl_ca.cert"
+  tls.mycert="/var/vcap/jobs/loghost_concentrator/config/ssl.cert"
+  tls.myprivkey="/var/vcap/jobs/loghost_concentrator/config/ssl.key"
+  tls.authmode="certvalid"
+  tls.permittedpeer="rsyslog"
+  <% end -%>
+)
+<% end -%>
+<% if p('loghost_concentrator.syslog.udp.enabled') -%>
 module(load="imudp")
 input(type="imudp" port="<%= p('loghost_concentrator.syslog.udp.port') %>")
-<% end %>
-
-<% if p('loghost_concentrator.syslog.tcp.enabled') %>
+<% end -%>
+<% if p('loghost_concentrator.syslog.tcp.enabled') -%>
 module(
   load="imtcp"
-  <% if_p('loghost_concentrator.syslog.tls.cert') do |v| %>
+  <% if_p('loghost_concentrator.syslog.tls.cert') do -%>
   StreamDriver.Name="gtls"
   StreamDriver.Mode="1"
   StreamDriver.Authmode="anon"
-  <% end %>
+  <% end -%>
 )
 input(type="imtcp" port="<%= p('loghost_concentrator.syslog.tcp.port') %>")
-<% end %>
+<% end -%>
 
 # load module that parses structured-data field
 module(load="mmpstrucdata")
@@ -79,26 +95,46 @@ if ($structured-data contains "instance@47450") then {
   # output to local file
   action(type="omfile" dynaFile="ParsedOutputFile" dirOwner="vcap" fileOwner="vcap" dirGroup="vcap" fileGroup="vcap" fileCreateMode="0640" dirCreateMode="0750")
 
-  # dupplicate output to local file for long-term logs
-  <% p('loghost_concentrator.syslog.longterm-jobs').each do |conf| %>
+  # duplicate output to local file for long-term logging
+<% p('loghost_concentrator.syslog.longterm-jobs').each do |conf| -%>
   if ($.deployment == "<%= conf['deployment'] %>") and ($.group == "<%= conf['job'] %>") then {
     action(type="omfile" dynaFile="ParsedOutputFileLT" dirOwner="vcap" fileOwner="vcap" dirGroup="vcap" fileGroup="vcap" fileCreateMode="0640" dirCreateMode="0750")
   }
-  <% end %>
+<% end -%>
 
-  <% if p('loghost_concentrator.syslog.forward').length > 0 %>
-  # extract structured-data into variables for easier use in condition expressions
-  <% p('loghost_concentrator.syslog.forward').each do |key, value| %>
+<% if p('loghost_concentrator.syslog.forward').length > 0 -%>
+  <%# extract structured-data into variables for easier use in condition expressions -%>
+  <% p('loghost_concentrator.syslog.forward').each do |_, value| -%>
   if <%= value.fetch("conditions").join(" and ") %> then {
-    <% value.fetch("targets").each do |forward| %>
-    action(type="omfwd" Target="<%= forward.fetch('address') %>" Port="<%= forward.fetch('port') %>" Protocol="<%= forward.fetch('transport') %>" Template="ForwardTemplate" <% if_p('loghost_concentrator.syslog.tls.cert') do |v| %> StreamDriver="gtls" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid" <% end %>)
-    <% end %>
-    <% if value.fetch("targets").empty? %>
+    <% value.fetch("targets").each do |forward| -%>
+      action(
+        Target="<%= forward.fetch('address') %>" Port="<%= forward.fetch('port') %>"
+        Template="ForwardTemplate"
+      <% if /relp/i =~ forward.fetch('transport') -%>
+        type="omrelp"
+        <% if_p('loghost_concentrator.syslog.tls.cert') do -%>
+        tls="on"
+        tls.cacert="/var/vcap/jobs/loghost_concentrator/config/ssl_ca.cert"
+        tls.mycert="/var/vcap/jobs/loghost_concentrator/config/ssl.cert"
+        tls.myprivkey="/var/vcap/jobs/loghost_concentrator/config/ssl.key"
+        tls.authmode="certvalid"
+        tls.permittedpeer="rsyslog"
+        tls.compression="on"
+        <% end -%>
+      <% else -%>
+        type="omfwd" Protocol="<%= forward.fetch('transport') %>"
+        <% if_p('loghost_concentrator.syslog.tls.cert') do -%>
+        StreamDriver="gtls" StreamDriverMode="1" StreamDriverAuthMode="x509/certvalid"
+        <% end %>
+      <% end -%>
+      )
+    <% end -%>
+    <% if value.fetch("targets").empty? -%>
     continue
-    <% end %>
+    <% end -%>
   }
-  <% end %>
-  <% end %>
+  <% end -%>
+<% end -%>
 
   # no further log processing
   stop