Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update logback.xml template for 2.0.0-M1 and make it optional #353

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Update logback.xml template for 2.0.0-M1 and make it optional #353

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8bafd45
Update logback.xml template for 2.0.0-M1 and make it optional
juldrixx Jan 11, 2024
8e184eb
Update CHANGELOG
juldrixx Jan 11, 2024
b24be6a
Update CHANGELOG
juldrixx Jan 11, 2024
acb4602
Merge branch 'update_logback_xml' of github.com:konpyutaika/nifikop i…
juldrixx Jan 11, 2024
041f81a
Update logback.xml template
juldrixx Jan 11, 2024
88478f7
Update helm chart logback.xml
juldrixx Jan 11, 2024
f3e3554
Revert
juldrixx Jan 11, 2024
9503527
Update helm chart logback.xml
juldrixx Jan 11, 2024
8114c23
Delete logback.xml template
juldrixx Jan 11, 2024
6f37246
Update CHANGELOG
juldrixx Jan 11, 2024
87f7d88
Update CHANGELOG
juldrixx Jan 11, 2024
5718661
Update doc
juldrixx Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
- [PR #349](https://github.com/konpyutaika/nifikop/pull/349) - **[Operator/NifiRegistryClient]** Set FlowRegistry type in RegistryClient creation.
- [PR #350](https://github.com/konpyutaika/nifikop/pull/350) - **[Operator]** Remove optimistic lock on `Patch`.
- [PR #352](https://github.com/konpyutaika/nifikop/pull/352) - **[Operator]** Changed default LogLevel of NiFi from `DEBUG` to `INFO`.
- [PR #353](https://github.com/konpyutaika/nifikop/pull/353) - **[Operator/NifiCluster]** Updated `logback.xml` template for 2.0.0-M1 and made it optional (by default disabled).

### Fixed Bugs

Expand Down
137 changes: 113 additions & 24 deletions helm/nifi-cluster/config/logback.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
-->

<configuration scan="true" scanPeriod="30 seconds">
<shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook" />

<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
Expand All @@ -29,9 +31,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxFileSize>5MB</maxFileSize>
<maxHistory>2</maxHistory>
<totalSizeCap>15MB</totalSizeCap>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
Expand All @@ -41,43 +48,112 @@

<appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'user_%d.log'.
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>2</maxHistory>
<maxFileSize>5MB</maxFileSize>
<totalSizeCap>15MB</totalSizeCap>
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>

<appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%msg%n</pattern>
</encoder>
</appender>

<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'bootstrap_%d.log'.
For hourly rollover, use 'bootstrap_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- keep 1 log files worth of history -->
<maxHistory>2</maxHistory>
<maxFileSize>5MB</maxFileSize>
<totalSizeCap>15MB</totalSizeCap>
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>

<appender name="DEPRECATION_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over is lower for deprecations -->
<maxFileSize>10MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>10</maxHistory>
<!-- Control the total size of all log archive files for this appender is lower as deprecations are repetitive -->
<totalSizeCap>100MB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger %msg%n</pattern>
</encoder>
</appender>

<appender name="DEDICATED_LOGGING" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator class="org.apache.nifi.logging.NifiDiscriminator"/>
<sift>
<appender name="APP-${logFileSuffix}_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--
For daily rollover, use 'app_%d.log'.
For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
Expand All @@ -86,12 +162,17 @@

<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->

<!-- Deprecation Log -->
<logger name="deprecation" level="WARN" additivity="false">
<appender-ref ref="DEPRECATION_FILE"/>
</logger>

<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="WARN"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />

<logger name="org.apache.parquet.hadoop.InternalParquetRecordReader" level="WARN" />

<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
Expand All @@ -100,8 +181,7 @@
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />

<logger name="org.apache.nifi.rocksdb.RocksDBMetronome" level="ERROR" />
<logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />

<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />

Expand All @@ -119,6 +199,7 @@

<!-- Suppress non-error messages due to excessive logging by class or library -->
<logger name="org.springframework" level="ERROR"/>
<logger name="org.springframework.security" level="INFO"/>

<!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
<logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>
Expand All @@ -134,11 +215,14 @@
<logger name="com.hierynomus.smbj" level="WARN" />

<!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
<logger name="com.amazonaws.services.kinesis" level="WARN" />
<logger name="software.amazon.awssdk.kinesis" level="WARN" />

<!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
<logger name="org.apache.atlas" level="WARN"/>

<!-- Suppress non-error messages from JetBrains Xodus FileDataWriter related to FileChannel -->
<logger name="jetbrains.exodus.io.FileDataWriter" level="WARN" />

<!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
<logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
<appender-ref ref="APP_FILE"/>
Expand All @@ -162,26 +246,26 @@
<logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.springframework.security.saml.log" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>

<!-- Web Server Request Log -->
<logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
<appender-ref ref="REQUEST_FILE"/>
</logger>

<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
-->
<logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>
<logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
<appender-ref ref="CONSOLE" />
mh013370 marked this conversation as resolved.
Show resolved Hide resolved
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>

Expand All @@ -202,4 +286,9 @@
<appender-ref ref="CONSOLE" />
</root>

<root level="INFO">
<appender-ref ref="DEDICATED_LOGGING" />
<appender-ref ref="CONSOLE" />
</root>

</configuration>
6 changes: 5 additions & 1 deletion pkg/nificlient/config/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (

"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/konpyutaika/nifikop/api/v1"
v1 "github.com/konpyutaika/nifikop/api/v1"
"github.com/konpyutaika/nifikop/pkg/pki"
"github.com/konpyutaika/nifikop/pkg/pki/certmanagerpki"
"github.com/konpyutaika/nifikop/pkg/util"
Expand Down Expand Up @@ -101,3 +101,7 @@ func generateNodesAddressFromTemplate(nodes []v1.Node, template string) map[int3
func UseSSL(cluster *v1.NifiCluster) bool {
return cluster.Spec.ListenersConfig.SSLSecrets != nil
}

func MustOverrideLogback(cluster *v1.NifiCluster) bool {
return false
}
24 changes: 17 additions & 7 deletions pkg/resources/nifi/secretconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas
"zookeeper.properties": []byte(r.generateZookeeperPropertiesNodeConfig(id, nodeConfig, log)),
"state-management.xml": []byte(r.getStateManagementConfigString(nodeConfig, id, log)),
"login-identity-providers.xml": []byte(r.getLoginIdentityProvidersConfigString(nodeConfig, id, log)),
"logback.xml": []byte(r.getLogbackConfigString(nodeConfig, id, log)),
"bootstrap.conf": []byte(r.generateBootstrapPropertiesNodeConfig(id, nodeConfig, log)),
"bootstrap-notification-services.xml": []byte(r.getBootstrapNotificationServicesConfigString(nodeConfig, id, log)),
},
Expand All @@ -52,6 +51,12 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas
if configcommon.UseSSL(r.NifiCluster) {
secret.Data["authorizers.xml"] = []byte(r.getAuthorizersConfigString(nodeConfig, id, log))
}

logbackConfigString := r.getLogbackConfigString(nodeConfig, id, log)
if logbackConfigString != nil {
secret.Data["logback.xml"] = []byte(*logbackConfigString)
}

return secret
}

Expand Down Expand Up @@ -310,13 +315,13 @@ func (r *Reconciler) getLoginIdentityProvidersConfigString(nConfig *v1.NodeConfi
// Logback configuration //
////////////////////////////

func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string {
func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) *string {
for _, node := range r.NifiCluster.Spec.Nodes {
if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.LogbackConfig != (&v1.LogbackConfig{}) {
if node.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig != nil {
conf, err := r.getSecrectConfig(context.TODO(), *node.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly secret config",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -327,7 +332,7 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if node.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap != nil {
conf, err := r.getConfigMap(context.TODO(), *node.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly configmap",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -341,7 +346,7 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig != nil {
conf, err := r.getSecrectConfig(context.TODO(), *r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly secret config",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -352,14 +357,18 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap != nil {
conf, err := r.getConfigMap(context.TODO(), *r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly configmap",
zap.String("clusterName", r.NifiCluster.Name),
zap.Int32("nodeId", id),
zap.Error(err))
}

if !configcommon.MustOverrideLogback(r.NifiCluster) {
juldrixx marked this conversation as resolved.
Show resolved Hide resolved
return nil
}

var out bytes.Buffer
t := template.Must(template.New("nConfig-config").Parse(config.LogbackTemplate))
if err := t.Execute(&out, map[string]interface{}{
Expand All @@ -371,7 +380,8 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
zap.Int32("nodeId", id),
zap.Error(err))
}
return out.String()
output := out.String()
return &output
}

///////////////////////////////////////////////////
Expand Down
Loading