Enrich crash email with detail and hints where crash reason can be inferred

forge/ee/lib/alerts/index.js

@@ -7,15 +7,34 @@ module.exports = {
         if (app.postoffice.enabled) {
             app.config.features.register('emailAlerts', true, true)
             app.auditLog.alerts = {}
-            app.auditLog.alerts.generate = async function (projectId, event) {
+            app.auditLog.alerts.generate = async function (projectId, event, data) {
                 if (app.postoffice.enabled) {
                     const project = await app.db.models.Project.byId(projectId)
                     const settings = await app.db.controllers.Project.getRuntimeSettings(project)
                     const teamType = await app.db.models.TeamType.byId(project.Team.TeamTypeId)
                     const emailAlerts = settings.emailAlerts
                     let template
                     if (emailAlerts?.crash && event === 'crashed') {
-                        template = 'Crashed'
+                        const templateName = ['Crashed']
+                        const hasLogs = data?.log?.length > 0
+                        let uncaughtException = false
+                        let outOfMemory = false
+                        if (hasLogs) {
+                            uncaughtException = data.exitCode > 0 && data.log.some(log => {
+                                const lcMsg = log.msg?.toLowerCase() || ''
+                                return lcMsg.includes('uncaughtexception') || log.msg.includes('uncaught exception')
+                            })
+                            outOfMemory = data.exitCode > 127 && data.log.some(log => {
+                                const lcMsg = log.msg?.toLowerCase() || ''
+                                return lcMsg.includes('heap out of memory') || lcMsg.includes('v8::internal::heap::')
+                            })
+                        }
+                        if (outOfMemory) {
+                            templateName.push('out-of-memory')
+                        } else if (uncaughtException) {
+                            templateName.push('uncaught-exception')
+                        }
+                        template = templateName.join('-')
                     } else if (emailAlerts?.safe && event === 'safe-mode') {
                         template = 'SafeMode'
                     }
@@ -42,9 +61,10 @@ module.exports = {
                         break
                     }
                     const users = (await app.db.models.TeamMember.findAll({ where, include: app.db.models.User })).map(tm => tm.User)
+                    const teamName = project.Team?.name || ''
                     if (users.length > 0) {
                         users.forEach(user => {
-                            app.postoffice.send(user, template, { name: project.name, url: `${app.config.base_url}/instance/${project.id}` })
+                            app.postoffice.send(user, template, { ...data, name: project.name, teamName, url: `${app.config.base_url}/instance/${project.id}` })
                         })
                     }
                 }

forge/postoffice/index.js

@@ -140,6 +140,50 @@ module.exports = fp(async function (app, _opts) {
             html: value.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/\./g, '<br style="display: none;"/>.')
         }
     }
+    /**
+     * Generates email-safe versions (both text and html) of a log array
+     * This is intended to make iso time strings and and sanitized log messages
+     * @param {Array<{ts: Number, level: String, msg: String}>} log
+     */
+    function sanitizeLog (log) {
+        const isoTime = (ts) => {
+            if (!ts) return ''
+            try {
+                let dt
+                if (typeof ts === 'string') {
+                    ts = +ts
+                }
+                // cater for ts with a 4 digit counter appended to the timestamp
+                if (ts > 99999999999999) {
+                    dt = new Date(ts / 10000)
+                } else {
+                    dt = new Date(ts)
+                }
+                let str = dt.toISOString().replace('T', ' ').replace('Z', '')
+                str = str.substring(0, str.length - 4) // remove milliseconds
+                return str
+            } catch (e) {
+                return ''
+            }
+        }
+        const htmlEscape = (str) => (str + '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+        return {
+            text: log.map(entry => {
+                return {
+                    timestamp: entry.ts ? isoTime(+entry.ts) : '',
+                    level: entry.level || '',
+                    message: entry.msg || ''
+                }
+            }),
+            html: log.map(entry => {
+                return {
+                    timestamp: entry.ts ? isoTime(+entry.ts) : '',
+                    level: htmlEscape(entry.level || ''),
+                    message: htmlEscape(entry.msg || '')
+                }
+            })
+        }
+    }
 
     /**
      * Send an email to a user
@@ -159,6 +203,11 @@ module.exports = fp(async function (app, _opts) {
         if (templateContext.invitee) {
             templateContext.invitee = sanitizeText(templateContext.invitee)
         }
+        if (Array.isArray(templateContext.log) && templateContext.log.length > 0) {
+            templateContext.log = sanitizeLog(templateContext.log)
+        } else {
+            delete templateContext.log
+        }
         const mail = {
             to: user.email,
             subject: template.subject(templateContext, { allowProtoPropertiesByDefault: true, allowProtoMethodsByDefault: true }),

forge/postoffice/layouts/default.js

@@ -22,7 +22,7 @@ module.exports = (htmlContent) => {
                 style="border-collapse:collapse;font-family:Helvetica,Arial,sans-serif;font-size:15px;color:#33475b;word-break:break-word;padding-top:20px;padding-bottom:20px">
                 <div style="color:inherit;font-size:inherit;line-height:inherit">
                     <div style="padding-left:10px;padding-right:10px">
-                        <div style="min-width:280px;max-width:600px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff;background-image:url('https://flowfuse.com/images/600x70-HS-newsletter-header.png');background-position:center;background-repeat:no-repeat;background-size:100% 100%"
+                        <div style="min-width:540px;max-width:1200px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff;background-image:url('https://flowfuse.com/images/600x70-HS-newsletter-header.png');background-position:center;background-repeat:no-repeat;background-size:100% 100%"
                              bgcolor="#ffffff">
                             <div>
                                 <div style="color:inherit;font-size:inherit;line-height:inherit">
@@ -46,7 +46,7 @@ module.exports = (htmlContent) => {
                         </div>
                     </div>
                     <div style="padding-left:10px;padding-right:10px">
-                        <div style="min-width:280px;max-width:600px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff"
+                        <div style="min-width:540px;max-width:1200px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff"
                              bgcolor="#FFFFFF">
                             <div>
                                 <table role="presentation" cellpadding="0" cellspacing="0" width="100%"
@@ -68,7 +68,7 @@ module.exports = (htmlContent) => {
                     </div>
 
                     <div style="padding-left:10px;padding-right:10px">
-                        <div style="min-width:280px;max-width:600px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff"
+                        <div style="min-width:540px;max-width:1200px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;background-color:#ffffff"
                              bgcolor="#FFFFFF">
                             <div>
                                 <table role="presentation" cellpadding="0" cellspacing="0" width="100%"
@@ -98,7 +98,7 @@ module.exports = (htmlContent) => {
                         </div>
                     </div>
                     <div style="padding-left:10px;padding-right:10px">
-                        <div style="min-width:280px;max-width:600px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;padding-bottom:20px">
+                        <div style="min-width:540px;max-width:1200px;Margin-left:auto;Margin-right:auto;border-collapse:collapse;border-spacing:0;padding-bottom:20px">
                             <div>
                                 <div style="color:inherit;font-size:inherit;line-height:inherit">
                                     <table role="presentation" width="100%" cellpadding="0" cellspacing="0"

forge/postoffice/templates/Crashed-out-of-memory.js

@@ -0,0 +1,86 @@
+module.exports = {
+    subject: 'FlowFuse Instance crashed',
+    text:
+`Hello
+
+Your FlowFuse Instance "{{{ name }}}" in Team "{{{ teamName.text }}}" has crashed due to an out of memory error.
+
+This can occur for a number of reasons including:
+- incorrect instance size for your workload
+- an issue in your flows or functions holding onto memory
+- an issue in a third-party library or node
+
+Possible solutions:
+- try selecting a larger instance type
+- try disabling some nodes to see if the problem settles down after a restart
+- when polling external services, ensure you are not polling too frequently as this may cause backpressure leading to memory exhaustion
+- check your flows for large data structures being held in memory, particularly in context
+- check the issue tracker of your contrib nodes
+
+{{#if log.text}}
+------------------------------------------------------
+Logs...
+
+{{#log.text}}
+Timestamp: {{{timestamp}}}
+Severity: {{{level}}}
+Message: {{{message}}}
+
+{{/log.text}}
+
+Note: Timestamps in this log are in UTC (Coordinated Universal Time).
+------------------------------------------------------
+{{/if}}
+
+You can access the instance and its logs here:
+
+{{{ url }}}
+
+`,
+    html:
+`<p>Hello</p>
+<p>Your FlowFuse Instance "{{{ name }}}" in Team "{{{ teamName.html }}}" has crashed due to an out of memory error.</p>
+
+<p>
+This can occur for a number of reasons including:
+<ul>
+<li>incorrect instance size for your workload/li>
+<li>an issue in your flows holding onto memory/li>
+<li>an issue in a third-party library or node/li>
+</ul>
+
+Possible solutions:
+<ul>
+<li>try selecting a larger instance type</li>
+<li>try disabling some nodes to see if the problem settles down after a restart</li>
+<li>when polling external services, ensure you are not polling too frequently as this may cause backpressure leading to memory exhaustion</li>
+<li>check your flows for large data structures being held in memory, particularly in context</li>
+<li>check the issue tracker of your contrib nodes</li>
+</p>
+
+
+{{#if log.html}}
+<p>
+Logs...
+<table style="width: 100%; font-size: small; font-family: monospace; white-space: pre;">
+    <tr>
+        <th style="text-align: left; min-width: 135px;">Timestamp</th>
+        <th style="text-align: left; white-space: nowrap;">Severity</th>
+        <th style="text-align: left;">Message</th>
+    </tr>
+    {{#log.html}}
+    <tr>
+        <td style="vertical-align: text-top;">{{{timestamp}}}</td>
+        <td style="vertical-align: text-top;">{{{level}}}</td>
+        <td>{{{message}}}</td>
+    </tr>
+    {{/log.html}}
+</table>
+<i>Note: Timestamps in this log are in UTC (Coordinated Universal Time).</i>
+</p>
+{{/if}}
+
+<p>You can access the instance and its logs here</p>
+<a href="{{{ url }}}">Instance Logs</a>
+`
+}

forge/postoffice/templates/Crashed-uncaught-exception.js

@@ -0,0 +1,81 @@
+module.exports = {
+    subject: 'FlowFuse Instance crashed',
+    text:
+`Hello
+
+Your FlowFuse Instance "{{{ name }}}" in Team "{{{ teamName.text }}}" has crashed due to an uncaught exception.
+
+This can occur for a number of reasons including:
+- an issue in your flows or function nodes
+- an issue in a third-party contribution node
+- an issue in Node-RED itself
+
+Possible solutions:
+- look out for async function calls in your function nodes that dont have error handling
+- check the issue tracker of the node that caused the crash
+- check the Node-RED issue tracker for similar issues
+
+{{#if log.text}}
+------------------------------------------------------
+Logs...
+
+{{#log.text}}
+Timestamp: {{{timestamp}}}
+Severity: {{{level}}}
+Message: {{{message}}}
+
+{{/log.text}}
+
+Note: Timestamps in this log are in UTC (Coordinated Universal Time).
+------------------------------------------------------
+{{/if}}
+
+You can access the instance and its logs here:
+
+{{{ url }}}
+
+`,
+    html:
+`<p>Hello</p>
+<p>Your FlowFuse Instance "{{{ name }}}" in Team "{{{ teamName.html }}}" has crashed due to an uncaught exception.</p>
+
+<p>
+This can occur for a number of reasons including:
+<ul>
+<li>an issue in your flows or function nodes</li>
+<li>an issue in a third-party contribution node</li>
+<li>an issue in Node-RED itself</li>
+</ul>
+
+Possible solutions:
+<ul>
+<li>look out for async function calls in your function nodes that dont have error handling</li>
+<li>check the issue tracker of the node that caused the crash</li>
+<li>check the Node-RED issue tracker for similar issues</li>
+</p>
+
+{{#if log.html}}
+<p>
+Logs...
+<table style="width: 100%; font-size: small; font-family: monospace; white-space: pre;">
+    <tr>
+        <th style="text-align: left; min-width: 135px;">Timestamp</th>
+        <th style="text-align: left; white-space: nowrap;">Severity</th>
+        <th style="text-align: left;">Message</th>
+    </tr>
+    {{#log.html}}
+    <tr>
+        <td style="vertical-align: text-top;">{{{timestamp}}}</td>
+        <td style="vertical-align: text-top;">{{{level}}}</td>
+        <td>{{{message}}}</td>
+    </tr>
+    {{/log.html}}
+</table>
+<i>Note: Timestamps in this log are in UTC (Coordinated Universal Time).</i>
+</p>
+{{/if}}
+
+<p>You can access the instance and its logs here</p>
+<a href="{{{ url }}}">Instance Logs</a>
+`
+}

forge/postoffice/templates/Crashed.js

@@ -3,18 +3,54 @@ module.exports = {
     text:
 `Hello
 
-Your FlowFuse Instance "{{{ name }}}" has crashed.
+Your FlowFuse Instance "{{{ name }}}"{{#if teamName.text}} in Team "{{{ teamName.text }}}"{{/if}} has crashed.
 
-You can access the logs here:
+{{#if log.text}}
+------------------------------------------------------
+Logs...
+
+{{#log.text}}
+Timestamp: {{{timestamp}}}
+Severity: {{{level}}}
+Message: {{{message}}}
+
+{{/log.text}}
+
+Note: Timestamps in this log are in UTC (Coordinated Universal Time).
+------------------------------------------------------
+{{/if}}
+
+You can access the instance and its logs here:
 
 {{{ url }}}
 
 `,
     html:
 `<p>Hello</p>
-<p>Your FlowFuse Instance "{{{ name }}}" has crashed</p>
+<p>Your FlowFuse Instance "{{{ name }}}"{{#if teamName.html}} in Team "{{{ teamName.html }}}"{{/if}} has crashed.</p>
+
+{{#if log.html}}
+<p>
+Logs...
+<table style="width: 100%; font-size: small; font-family: monospace; white-space: pre;">
+    <tr>
+        <th style="text-align: left; min-width: 135px;">Timestamp</th>
+        <th style="text-align: left; white-space: nowrap;">Severity</th>
+        <th style="text-align: left;">Message</th>
+    </tr>
+    {{#log.html}}
+    <tr>
+        <td style="vertical-align: text-top;">{{{timestamp}}}</td>
+        <td style="vertical-align: text-top;">{{{level}}}</td>
+        <td>{{{message}}}</td>
+    </tr>
+    {{/log.html}}
+</table>
+<i>Note: Timestamps in this log are in UTC (Coordinated Universal Time).</i>
+</p>
+{{/if}}
 
-<p>You can access the logs here</p>
+<p>You can access the instance and its logs here</p>
 <a href="{{{ url }}}">Instance Logs</a>
 `
 }