Merge pull request #80 from axoflow/updates-for-4.9

Updates for 4.9
axoflow · Nov 12, 2024 · 16145e4 · 16145e4
2 parents 49e5070 + 98fcc82
commit 16145e4
Show file tree

Hide file tree

Showing 22 changed files with 374 additions and 48 deletions.
diff --git a/config/_default/config.toml b/config/_default/config.toml
@@ -66,7 +66,7 @@ description = "Documentation for AxoSyslog, the scalable security data processor
   # The version number for the version of the docs represented in this doc set.
   # Used in the "version-banner" partial to display a version number for the 
   # current doc set.
-  version = "4.8.0"
+  version = "4.9.0"
   version_menu_canonicallinks = true
 
   # A link to latest version of the docs. Used in the "version-banner" partial to
@@ -160,9 +160,9 @@ description = "Documentation for AxoSyslog, the scalable security data processor
 [params.product]
 name = "AxoSyslog"
 abbrev = "AxoSyslog"
-version = "4.8"
-techversion = "4.8.1"
-configversion = "4.8"
+version = "4.9"
+techversion = "4.9.0"
+configversion = "4.9"
 syslog-ng = "syslog-ng"
 selinux = "SELinux"
 apparmor = "AppArmor"

diff --git a/content/app-man-syslog-ng/syslog-ng-ctl.1.md b/content/app-man-syslog-ng/syslog-ng-ctl.1.md
@@ -63,6 +63,7 @@ If you need to use a non-standard control socket to access `syslog-ng`, use the
 syslog-ng-ctl log-level verbose
 ```
 
+To temporarily change the log levels and access the logs of `syslog-ng`, see also the [`attach` command]({{< relref "#attach" >}}).
 
 <span id="syslog-ng-ctl-query"></span>
 
@@ -371,6 +372,40 @@ You can run `syslog-ng-ctl healthcheck --timeout <seconds>` to use as a boolean
 
 Health checks are also published as periodically updated metrics. You can configure the frequency of these checks with the `stats(healthcheck-freq())` option. The default is 5 minutes.
 
+## The attach command {#attach}
+
+Available in {{% param "product.abbrev" %}} 4.9 and later.
+
+Connect to the standard IO (stdin, stdout, stderr) and display the results. Note that there can only be one attached process at a time.
+
+`syslog-ng-ctl attach [attach-mode] [options]`
+
+The `syslog-ng-ctl attach` command has the following parameters:
+
+- Attach mode: `logs` or `stdio`.
+
+    - Use `logs` to access the internal log messages of `syslog-ng`. For example, the following command changes the log level to `trace` and accesses the internal logs of `syslog-ng`:
+
+        ```shell
+        syslog-ng-ctl attach logs --seconds 10 --log-level trace
+        ```
+
+    - Use `stdio` to display the output of the `syslog-ng` process. For example:
+
+        ```shell
+        syslog-ng-ctl attach stdio --seconds 10
+        ```
+
+- Change `log-level` to the specified value:
+
+    {{< include-headless "chunk/internal-log-levels.md" >}}
+
+- How long to attach to the process: `--seconds`. For example:
+
+    ```shell
+    syslog-ng-ctl attach stdio --seconds 10
+    ```
+
 ## Files
 
 `/opt/syslog-ng/sbin/syslog-ng-ctl`

diff --git a/content/chapter-destinations/clickhouse/_index.md b/content/chapter-destinations/clickhouse/_index.md
@@ -0,0 +1,198 @@
+---
+title: ClickHouse database
+linktitle: "ClickHouse"
+weight:  200
+driver: "clickhouse()"
+short_description: "Send messages to a ClickHouse database"
+---
+<!-- This file is under the copyright of Axoflow, and licensed under Apache License 2.0, except for using the Axoflow and AxoSyslog trademarks. -->
+
+Starting with version 4.9.0, {{% param "product_name" %}} can send data to [ClickHouse databases](https://clickhouse.com/) using its [gRPC interface](https://clickhouse.com/docs/en/interfaces/grpc).
+
+## Prerequisites
+
+- A [self-hosted ClickHouse installation](https://clickhouse.com/docs/en/install).
+
+    {{% alert title="Warning" color="warning" %}}
+ClickHouse Cloud doesn't support the gRPC interface currently.
+    {{% /alert %}}
+
+- The [gRPC interface](https://clickhouse.com/docs/en/interfaces/grpc) must be enabled in your ClickHouse configuration.
+- To configure {{% param "product_name" %}}, you'll need:
+
+    - the name of an existing database and a table where you want to send your data, and
+    - the credentials (username and password) to access the database.
+
+Example configuration (sends data to the default `localhost:9100` URL):
+
+```sh
+destination {
+  clickhouse(
+    database("default")
+    table("demo_table")
+    user("your-username")
+    password("your-password")
+    schema(
+      "user_id" UInt32 => $R_MSEC,
+      "message" String => "$MSG",
+      "timestamp" DateTime => "$R_UNIXTIME",
+      "metric" Float32 => 3.14
+    )
+  );
+};
+```
+
+## Options
+
+This destination has the following options:
+
+{{< include-headless "chunk/grpc-authentication.md" >}}
+
+{{% include-headless "chunk/option-destination-otlp-batch-bytes.md" %}}
+
+## batch-lines()
+
+|          |        |
+| -------- | ------ |
+| Type:    | number |
+| Default: | 0      |
+
+{{% include-headless "chunk/option-description-destination-batch-lines.md" %}}
+
+{{% include-headless "chunk/option-destination-batch-timeout.md" %}}
+
+{{< include-headless "chunk/option-grpc-channel-args.md" >}}
+
+{{< include-headless "chunk/option-destination-grpc-compression.md" >}}
+
+## database()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | string |
+| Default: | `default` |
+
+*Description:* The database where {{% param "product_name" %}} sends the data.
+
+{{< include-headless "chunk/option-destination-diskbuffer.md" >}}
+
+{{< include-headless "chunk/option-destination-frac-digits.md" >}}
+
+{{< include-headless "chunk/option-grpc-headers.md" >}}
+
+{{< include-headless "chunk/option-destination-hook.md" >}}
+
+{{< include-headless "chunk/option-destination-grpc-keep-alive.md" >}}
+
+{{% include-headless "chunk/option-destination-local-timezone.md" %}}
+
+{{% include-headless "chunk/option-destination-log-fifo-size.md" %}}
+
+{{< include-headless "chunk/option-destination-on-error.md" >}}
+
+## password()
+
+|          |              |
+| -------- | ------------ |
+| Type:    | string       |
+| Default: | empty string |
+
+*Description:* The password used for authentication.
+
+{{% include-headless "chunk/option-persist-name.md" %}}
+
+## protobuf-schema()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | See the description |
+| Default: | - |
+
+*Description:* Sets the schema of the database table from a protobuf schema file.
+
+```shell
+protobuf-schema("/tmp/test.proto" => "$MESSAGE", "$PROGRAM", "$HOST", "$PID")
+```
+
+An example proto file when using the `protobuf-schema()` option:
+
+```shell
+syntax = "proto2";
+
+message CustomRecord {
+  optional string message = 1;
+  optional string app = 2;
+  optional string host = 3;
+  optional int64 pid = 4;
+}
+```
+
+Alternatively, you can set the schema with the [`schema()`](#schema) option.
+
+{{% include-headless "chunk/option-destination-retries.md" %}}
+
+## schema()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | arrow list |
+| Default: |  |
+
+*Description:* Sets the schema of the database table. On the left side of the arrow, set the name of the column and its type. On the right side, set any {{% param "product_name" %}} template or macro, which gets evaluated on each log that is routed to the destination. For example:
+
+```shell
+schema(
+  "user_id" UInt32 => $R_MSEC,
+  "message" String => "$MSG",
+  "timestamp" DateTime => "$R_UNIXTIME",
+  "metric" Float32 => 3.14
+)
+```
+
+Alternatively, you can set the schema with the [`protobuf-schema()`](#protobuf-schema) option.
+
+You can find the available column types in the [official ClickHouse documentation](https://clickhouse.com/docs/en/sql-reference/data-types).
+
+{{% include-headless "chunk/option-destination-send-timezone.md" %}}
+
+## table()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | string |
+| Default: | - |
+
+*Description:* The name of the table where {{% param "product_name" %}} sends the data.
+
+{{% include-headless "chunk/option-destination-template-escape.md" %}}
+
+{{% include-headless "chunk/option-destination-throttle.md" %}}
+
+{{% include-headless "chunk/option-source-time-reopen.md" %}}
+
+{{% include-headless "chunk/option-destination-timezone.md" %}}
+
+{{< include-headless "chunk/option-destination-ts-format.md" >}}
+
+## url()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | string |
+| Default: | `localhost:9100` |
+
+*Description:* The URL of the gRPC receiver.
+
+## user()
+
+|          |              |
+| -------- | ------------ |
+| Type:    | string       |
+| Default: | empty string |
+
+*Description:* The username used for authentication.
+
+<a id="worker-partition-key"></a>
+{{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}}
+
+{{< include-headless "chunk/option-destination-threaded-workers.md" >}}
diff --git a/content/chapter-destinations/destination-loki/_index.md b/content/chapter-destinations/destination-loki/_index.md
@@ -35,6 +35,8 @@ The `loki()` destination has the following options.
 
 {{< include-headless "chunk/grpc-authentication.md" >}}
 
+{{% include-headless "chunk/option-destination-batch-bytes.md" %}}
+
 ## batch-lines()
 
 |          |        |
@@ -48,6 +50,8 @@ The `loki()` destination has the following options.
 
 {{< include-headless "chunk/option-grpc-channel-args.md" >}}
 
+{{< include-headless "chunk/option-destination-grpc-compression.md" >}}
+
 {{< include-headless "chunk/option-grpc-headers.md" >}}
 
 {{< include-headless "chunk/option-destination-grpc-keep-alive.md" >}}

diff --git a/content/chapter-destinations/destination-syslog-ng-otlp/_index.md b/content/chapter-destinations/destination-syslog-ng-otlp/_index.md
@@ -37,18 +37,32 @@ The `syslog-ng-otlp()` destination has the following options.
 
 {{< include-headless "chunk/option-destination-diskbuffer.md" >}}
 
+{{< include-headless "chunk/option-destination-frac-digits.md" >}}
+
 {{< include-headless "chunk/option-destination-hook.md" >}}
 
+{{% include-headless "chunk/option-destination-local-timezone.md" %}}
+
 {{% include-headless "chunk/option-destination-log-fifo-size.md" %}}
 
+{{< include-headless "chunk/option-destination-on-error.md" >}}
+
 {{% include-headless "chunk/option-persist-name.md" %}}
 
 {{% include-headless "chunk/option-destination-retries.md" %}}
 
+{{% include-headless "chunk/option-destination-send-timezone.md" %}}
+
+{{% include-headless "chunk/option-destination-template-escape.md" %}}
+
 {{% include-headless "chunk/option-destination-throttle.md" %}}
 
 {{% include-headless "chunk/option-source-time-reopen.md" %}}
 
+{{% include-headless "chunk/option-destination-timezone.md" %}}
+
+{{< include-headless "chunk/option-destination-ts-format.md" >}}
+
 ## url()
 
 |          |                            |

diff --git a/content/chapter-destinations/google-bigquery/_index.md b/content/chapter-destinations/google-bigquery/_index.md
@@ -56,6 +56,8 @@ By default, the messages are sent with one worker, one message per batch, and wi
 
 The `bigquery()` destination has the following options.
 
+{{< include-headless "chunk/grpc-authentication.md" >}}
+
 {{% include-headless "chunk/option-destination-batch-bytes.md" %}}
 
 By default, the `batch-bytes()` option of the `bigquery()` destination is 10 MB. This is an upper limit for the `bigquery()` destination. Note that due to a framework limitation, the batch might be at most 1 message larger than the set limit.

diff --git a/content/chapter-destinations/opentelemetry/_index.md b/content/chapter-destinations/opentelemetry/_index.md
@@ -68,9 +68,46 @@ log non_otel_to_otel_tls {
 
 {{< include-headless "chunk/option-destination-grpc-compression.md" >}}
 
+{{< include-headless "chunk/option-destination-diskbuffer.md" >}}
+
 {{< include-headless "chunk/option-grpc-headers.md" >}}
 
-{{< include-headless "chunk/option-destination-threaded-workers.md" >}}
+{{< include-headless "chunk/option-destination-frac-digits.md" >}}
+
+{{< include-headless "chunk/option-destination-hook.md" >}}
+
+{{% include-headless "chunk/option-destination-local-timezone.md" %}}
+
+{{% include-headless "chunk/option-destination-log-fifo-size.md" %}}
+
+{{< include-headless "chunk/option-destination-on-error.md" >}}
+
+{{% include-headless "chunk/option-persist-name.md" %}}
+
+{{% include-headless "chunk/option-destination-retries.md" %}}
+
+{{% include-headless "chunk/option-destination-send-timezone.md" %}}
+
+{{% include-headless "chunk/option-destination-template-escape.md" %}}
 
-<!-- FIXME xinclude other common options
- threaded_dest_driver_general_option-->
+{{% include-headless "chunk/option-destination-throttle.md" %}}
+
+{{% include-headless "chunk/option-source-time-reopen.md" %}}
+
+{{% include-headless "chunk/option-destination-timezone.md" %}}
+
+{{< include-headless "chunk/option-destination-ts-format.md" >}}
+
+## url()
+
+|          |                            |
+| -------- | -------------------------- |
+| Type:    | string |
+| Default: | `localhost:9095` |
+
+*Description:* The URL of the OpenTelemetry receiver.
+
+<a id="worker-partition-key"></a>
+{{< include-headless "chunk/option-destination-http-worker-partition-key.md" >}}
+
+{{< include-headless "chunk/option-destination-threaded-workers.md" >}}
diff --git a/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md b/content/chapter-sources/configuring-sources-file/reference-source-file/_index.md
@@ -20,6 +20,8 @@ The `file()` driver has the following options:
 
 {{< include-headless "chunk/option-destination-hook.md" >}}
 
+{{< include-headless "chunk/option-source-idle-timeout.md" >}}
+
 {{< include-headless "chunk/option-source-keep-timestamp.md" >}}
 
 {{< include-headless "chunk/option-source-log-fetch-limit.md" >}}

diff --git a/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md b/content/chapter-sources/configuring-sources-stdin/stdin()-source-options/_index.md
@@ -20,6 +20,8 @@ The `stdin()` driver has the following options:
 
 {{< include-headless "chunk/option-destination-hook.md" >}}
 
+{{< include-headless "chunk/option-source-idle-timeout.md" >}}
+
 {{< include-headless "chunk/option-source-keep-timestamp.md" >}}
 
 {{% include-headless "chunk/option-source-log-fetch-limit.md" %}}