Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update logback.xml template for 2.0.0-M1 and make it optional #353

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Update logback.xml template for 2.0.0-M1 and make it optional #353

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
8bafd45
Update logback.xml template for 2.0.0-M1 and make it optional
juldrixx Jan 11, 2024
8e184eb
Update CHANGELOG
juldrixx Jan 11, 2024
b24be6a
Update CHANGELOG
juldrixx Jan 11, 2024
acb4602
Merge branch 'update_logback_xml' of github.com:konpyutaika/nifikop i…
juldrixx Jan 11, 2024
041f81a
Update logback.xml template
juldrixx Jan 11, 2024
88478f7
Update helm chart logback.xml
juldrixx Jan 11, 2024
f3e3554
Revert
juldrixx Jan 11, 2024
9503527
Update helm chart logback.xml
juldrixx Jan 11, 2024
8114c23
Delete logback.xml template
juldrixx Jan 11, 2024
6f37246
Update CHANGELOG
juldrixx Jan 11, 2024
87f7d88
Update CHANGELOG
juldrixx Jan 11, 2024
5718661
Update doc
juldrixx Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
- [PR #349](https://github.com/konpyutaika/nifikop/pull/349) - **[Operator/NifiRegistryClient]** Set FlowRegistry type in RegistryClient creation.
- [PR #350](https://github.com/konpyutaika/nifikop/pull/350) - **[Operator]** Remove optimistic lock on `Patch`.
- [PR #352](https://github.com/konpyutaika/nifikop/pull/352) - **[Operator]** Changed default LogLevel of NiFi from `DEBUG` to `INFO`.
- [PR #353](https://github.com/konpyutaika/nifikop/pull/353) - **[Operator/NifiCluster]** Updated `logback.xml` template for 2.0.0-M1 and made it optional (by default disabled).

### Fixed Bugs

Expand Down
6 changes: 5 additions & 1 deletion pkg/nificlient/config/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (

"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/konpyutaika/nifikop/api/v1"
v1 "github.com/konpyutaika/nifikop/api/v1"
"github.com/konpyutaika/nifikop/pkg/pki"
"github.com/konpyutaika/nifikop/pkg/pki/certmanagerpki"
"github.com/konpyutaika/nifikop/pkg/util"
Expand Down Expand Up @@ -101,3 +101,7 @@ func generateNodesAddressFromTemplate(nodes []v1.Node, template string) map[int3
func UseSSL(cluster *v1.NifiCluster) bool {
return cluster.Spec.ListenersConfig.SSLSecrets != nil
}

func MustOverrideLogback(cluster *v1.NifiCluster) bool {
return false
}
24 changes: 17 additions & 7 deletions pkg/resources/nifi/secretconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas
"zookeeper.properties": []byte(r.generateZookeeperPropertiesNodeConfig(id, nodeConfig, log)),
"state-management.xml": []byte(r.getStateManagementConfigString(nodeConfig, id, log)),
"login-identity-providers.xml": []byte(r.getLoginIdentityProvidersConfigString(nodeConfig, id, log)),
"logback.xml": []byte(r.getLogbackConfigString(nodeConfig, id, log)),
"bootstrap.conf": []byte(r.generateBootstrapPropertiesNodeConfig(id, nodeConfig, log)),
"bootstrap-notification-services.xml": []byte(r.getBootstrapNotificationServicesConfigString(nodeConfig, id, log)),
},
Expand All @@ -52,6 +51,12 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas
if configcommon.UseSSL(r.NifiCluster) {
secret.Data["authorizers.xml"] = []byte(r.getAuthorizersConfigString(nodeConfig, id, log))
}

logbackConfigString := r.getLogbackConfigString(nodeConfig, id, log)
if logbackConfigString != nil {
secret.Data["logback.xml"] = []byte(*logbackConfigString)
}

return secret
}

Expand Down Expand Up @@ -310,13 +315,13 @@ func (r *Reconciler) getLoginIdentityProvidersConfigString(nConfig *v1.NodeConfi
// Logback configuration //
////////////////////////////

func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string {
func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) *string {
for _, node := range r.NifiCluster.Spec.Nodes {
if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.LogbackConfig != (&v1.LogbackConfig{}) {
if node.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig != nil {
conf, err := r.getSecrectConfig(context.TODO(), *node.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly secret config",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -327,7 +332,7 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if node.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap != nil {
conf, err := r.getConfigMap(context.TODO(), *node.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly configmap",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -341,7 +346,7 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig != nil {
conf, err := r.getSecrectConfig(context.TODO(), *r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceSecretConfig)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly secret config",
zap.String("clusterName", r.NifiCluster.Name),
Expand All @@ -352,14 +357,18 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
if r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap != nil {
conf, err := r.getConfigMap(context.TODO(), *r.NifiCluster.Spec.ReadOnlyConfig.LogbackConfig.ReplaceConfigMap)
if err == nil {
return conf
return &conf
}
log.Error("error occurred during getting readonly configmap",
zap.String("clusterName", r.NifiCluster.Name),
zap.Int32("nodeId", id),
zap.Error(err))
}

if !configcommon.MustOverrideLogback(r.NifiCluster) {
juldrixx marked this conversation as resolved.
Show resolved Hide resolved
return nil
}

var out bytes.Buffer
t := template.Must(template.New("nConfig-config").Parse(config.LogbackTemplate))
if err := t.Execute(&out, map[string]interface{}{
Expand All @@ -371,7 +380,8 @@ func (r *Reconciler) getLogbackConfigString(nConfig *v1.NodeConfig, id int32, lo
zap.Int32("nodeId", id),
zap.Error(err))
}
return out.String()
output := out.String()
return &output
}

///////////////////////////////////////////////////
Expand Down
162 changes: 142 additions & 20 deletions pkg/resources/templates/config/logback.xml.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>
See the License for the specific language governing permissions and
limitations under the License.
-->

<configuration scan="true" scanPeriod="30 seconds">
<shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook" />

<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>

<appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
Expand All @@ -30,16 +33,21 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history -->
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>

<appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
Expand All @@ -50,59 +58,138 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
<!-- keep 30 log files worth of history -->
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>

<appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%msg%n</pattern>
</encoder>
</appender>

<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!--
For daily rollover, use 'user_%d.log'.
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
For daily rollover, use 'bootstrap_%d.log'.
For hourly rollover, use 'bootstrap_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history -->
<maxHistory>5</maxHistory>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>


<appender name="DEPRECATION_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over is lower for deprecations -->
<maxFileSize>10MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>10</maxHistory>
<!-- Control the total size of all log archive files for this appender is lower as deprecations are repetitive -->
<totalSizeCap>100MB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger %msg%n</pattern>
</encoder>
</appender>

<appender name="DEDICATED_LOGGING" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator class="org.apache.nifi.logging.NifiDiscriminator"/>
<sift>
<appender name="APP-${logFileSuffix}_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--
For daily rollover, use 'app_%d.log'.
For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
To GZIP rolled files, replace '.log' with '.log.gz'.
To ZIP rolled files, replace '.log' with '.log.zip'.
-->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>
</sift>
</appender>

<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
</encoder>
</appender>

<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->


<!-- Deprecation Log -->
<logger name="deprecation" level="WARN" additivity="false">
<appender-ref ref="DEPRECATION_FILE"/>
</logger>

<logger name="org.apache.nifi" level="INFO"/>
<logger name="org.apache.nifi.processors" level="WARN"/>
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />

<logger name="org.apache.parquet.hadoop.InternalParquetRecordReader" level="WARN" />

<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" />
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
<logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />

<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />

<logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />
<logger name="org.apache.curator.ConnectionState" level="OFF" />

<!-- Logger for managing logging statements for nifi clusters. -->
<logger name="org.apache.nifi.cluster" level="INFO"/>

Expand All @@ -114,10 +201,35 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>

<!-- Suppress non-error messages due to excessive logging by class or library -->
<logger name="org.springframework" level="ERROR"/>

<logger name="org.springframework.security" level="INFO"/>

<!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
<logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>

<!-- Suppress non-error messages due to Jetty AnnotationParser emitting a large amount of WARNS. Issue described in NIFI-5479. -->
<logger name="org.eclipse.jetty.annotations.AnnotationParser" level="ERROR"/>

<!-- Suppress non-error messages from SSHJ which was emitting large amounts of INFO logs by default -->
<logger name="net.schmizz.sshj" level="WARN" />
<logger name="com.hierynomus.sshj" level="WARN" />

<!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default -->
<logger name="com.hierynomus.smbj" level="WARN" />

<!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
<logger name="software.amazon.awssdk.kinesis" level="WARN" />

<!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
<logger name="org.apache.atlas" level="WARN"/>

<!-- Suppress non-error messages from JetBrains Xodus FileDataWriter related to FileChannel -->
<logger name="jetbrains.exodus.io.FileDataWriter" level="WARN" />

<!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
<logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
<appender-ref ref="APP_FILE"/>
</logger>

<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the
Expand All @@ -135,13 +247,20 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>
<logger name="org.opensaml" level="WARN" additivity="false">
<appender-ref ref="USER_FILE"/>
</logger>

<!-- Web Server Request Log -->
<logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
<appender-ref ref="REQUEST_FILE"/>
</logger>

<!--
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
-->
<logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
Expand All @@ -155,16 +274,19 @@ var LogbackTemplate = `<?xml version="1.0" encoding="UTF-8"?>
<logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>

<!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
<logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
<appender-ref ref="BOOTSTRAP_FILE" />
</logger>

<root level="INFO">
<appender-ref ref="APP_FILE" />
</root>

<root level="INFO">
<appender-ref ref="APP_FILE"/>
<appender-ref ref="DEDICATED_LOGGING" />
</root>

</configuration>
`