fix: Refactor progress extraction and add tests

backend/src/api/pod-progress.ts

@@ -2,16 +2,9 @@ import { FastifyPluginAsync } from 'fastify'
 import { Controllers } from '../controllers.js'
 import { forbidden, notFound } from './errors.js'
 import { authenticateSession } from '../auth/common.js'
-import sjson from 'secure-json-parse'
+import { extractProgress, ProgressItem } from '../renovate/progress.js'
 import { BackendConfig } from '../backend-config.js'
 
-interface ProgressItem {
-  repository: string
-  repositoryUrl?: string
-  state: 'pending' | 'started' | 'finished'
-  duration?: number
-}
-
 export interface PodProgressRoute {
   Reply: ProgressItem[]
 }
@@ -32,77 +25,11 @@ export const podProgressRoute = ({ logsController }: Controllers, config: Backen
     if (logs == null) {
       return await notFound(reply)
     }
-    const progress = getProgress(logs, config)
+    const progress = extractProgress(logs, { repositoryBaseUrl: config.gitlab.host })
     if (progress == null) {
       // In case the log cannot be parsed, progress is unavailable, so return a 404.
       return await notFound((reply))
     }
     return progress
   })
 }
-
-function getProgress (logs: string, config: BackendConfig): ProgressItem[] | undefined {
-  const repositories = new Map<string, ProgressItem>()
-  let foundRepositories = false
-  for (const line of lines(logs)) {
-    const message = tryParseLogMessage(line)
-    // Note: While it looks ugly, manual validation is more than 3x faster than using superstruct.
-    //       This is worth it here as there is lots of data to process.
-    if (message == null || typeof message !== 'object' || !('msg' in message)) {
-      continue
-    }
-    // The initial log message is for autodiscovery and tells us about the repositories that will be processed.
-    if ('repositories' in message && Array.isArray(message.repositories) &&
-      message.repositories.every(msg => typeof msg === 'string')) {
-      foundRepositories = true
-      for (const repository of message.repositories) {
-        const item: ProgressItem = { repository, state: 'pending' }
-        if (config.gitlab.host != null) {
-          item.repositoryUrl = new URL(repository, config.gitlab.host).toString()
-        }
-        repositories.set(repository, item)
-      }
-    }
-    // Log messages associated with a repository have a repository property, and update the item's status.
-    if (!('repository' in message) || typeof message.repository !== 'string') {
-      continue
-    }
-    const item = repositories.get(message.repository)
-    if (item == null) {
-      continue // what?
-    }
-    switch (message.msg) {
-      case 'Repository started':
-        item.state = 'started'
-        break
-      case 'Repository finished':
-        item.state = 'finished'
-        break
-    }
-  }
-  if (!foundRepositories) {
-    return undefined
-  }
-  // Note: JS Map always returns values in insertion order.
-  return Array.from(repositories.values())
-}
-
-function * lines (str: string): Generator<string> {
-  let lineFrom = 0
-  while (lineFrom < str.length) {
-    let lineTo = str.indexOf('\n', lineFrom)
-    if (lineTo === -1) {
-      lineTo = str.length
-    }
-    yield str.slice(lineFrom, lineTo)
-    lineFrom = lineTo + 1
-  }
-}
-
-function tryParseLogMessage (str: string): unknown {
-  try {
-    return sjson.parse(str)
-  } catch {
-    return undefined
-  }
-}

backend/src/renovate/progress.ts

@@ -0,0 +1,75 @@
+import sjson from 'secure-json-parse'
+
+export interface ProgressItem {
+  repository: string
+  repositoryUrl?: string
+  state: 'pending' | 'started' | 'finished'
+}
+
+export function extractProgress (logs: string, options: {
+  repositoryBaseUrl?: string
+}): ProgressItem[] | undefined {
+  const repositories = new Map<string, ProgressItem>()
+  let foundRepositories = false
+  for (const line of lines(logs)) {
+    const message = tryParseLogMessage(line)
+    // Note: While it looks ugly, manual validation is more than 3x faster than using superstruct.
+    //       This is worth it here as there is lots of data to process.
+    if (message == null || typeof message !== 'object' || !('msg' in message)) {
+      continue
+    }
+    // The initial log message is for autodiscovery and tells us about the repositories that will be processed.
+    if ('repositories' in message && Array.isArray(message.repositories) &&
+      message.repositories.every(msg => typeof msg === 'string')) {
+      foundRepositories = true
+      for (const repository of message.repositories) {
+        const item: ProgressItem = { repository, state: 'pending' }
+        if (options.repositoryBaseUrl != null) {
+          item.repositoryUrl = new URL(repository, options.repositoryBaseUrl).toString()
+        }
+        repositories.set(repository, item)
+      }
+    }
+    // Log messages associated with a repository have a repository property, and update the item's status.
+    if (!('repository' in message) || typeof message.repository !== 'string') {
+      continue
+    }
+    const item = repositories.get(message.repository)
+    if (item == null) {
+      continue // what?
+    }
+    switch (message.msg) {
+      case 'Repository started':
+        item.state = 'started'
+        break
+      case 'Repository finished':
+        item.state = 'finished'
+        break
+    }
+  }
+  if (!foundRepositories) {
+    return undefined
+  }
+  // Note: JS Map always returns values in insertion order.
+  return Array.from(repositories.values())
+}
+
+function * lines (str: string): Generator<string> {
+  let lineFrom = 0
+  while (lineFrom < str.length) {
+    let lineTo = str.indexOf('\n', lineFrom)
+    if (lineTo === -1) {
+      lineTo = str.length
+    }
+    yield str.slice(lineFrom, lineTo)
+    lineFrom = lineTo + 1
+  }
+}
+
+function tryParseLogMessage (str: string): unknown {
+  try {
+    return sjson.parse(str)
+  } catch {
+    return undefined
+  }
+}

backend/test/renovate/progress.test.ts

@@ -0,0 +1,122 @@
+import assert from 'node:assert'
+import { extractProgress } from '../../src/renovate/progress.js'
+
+describe('renovate/progress.ts', () => {
+  describe('extractProgress()', () => {
+    it('returns undefined for empty input', async () => {
+      const result = extractProgress('', {})
+      assert.strictEqual(result, undefined)
+    })
+
+    it('ignores non-JSON input', async () => {
+      const result = extractProgress('foo bar\nbaz qux 42 {}', {})
+      assert.strictEqual(result, undefined)
+    })
+
+    it('detects repository list', async () => {
+      const log = [
+        '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}',
+        '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}'
+      ].join('\n')
+      const result = extractProgress(log, {})
+      assert.deepStrictEqual(result, [
+        {
+          repository: 'foo/bar',
+          state: 'pending'
+        },
+        {
+          repository: 'foo/baz/qux',
+          state: 'pending'
+        },
+        {
+          repository: 'random',
+          state: 'pending'
+        }
+      ])
+    })
+
+    it('detects repository started/finished', async () => {
+      const log = [
+        '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}',
+        '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}',
+        '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}',
+        '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","cloned":true,"durationMs":7133,"msg":"Repository finished","time":"2024-07-12T17:00:30.123Z","v":0}',
+        '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/baz/qux","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:32.456Z","v":0}'
+      ].join('\n')
+      const result = extractProgress(log, {})
+      assert.deepStrictEqual(result, [
+        {
+          repository: 'foo/bar',
+          state: 'finished'
+        },
+        {
+          repository: 'foo/baz/qux',
+          state: 'started'
+        },
+        {
+          repository: 'random',
+          state: 'pending'
+        }
+      ])
+    })
+  })
+
+  it('ignores invalid log lines', async () => {
+    const log = [
+      '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}',
+      'null',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}',
+      'false',
+      'true',
+      '{}',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}',
+      '[{"msg:"something"}]',
+      'some random text',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","cloned":true,"durationMs":7133,"msg":"Repository finished","time":"2024-07-12T17:00:30.123Z","v":0}',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/baz/qux","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:32.456Z","v":0}'
+    ].join('\n')
+    const result = extractProgress(log, {})
+    assert.deepStrictEqual(result, [
+      {
+        repository: 'foo/bar',
+        state: 'finished'
+      },
+      {
+        repository: 'foo/baz/qux',
+        state: 'started'
+      },
+      {
+        repository: 'random',
+        state: 'pending'
+      }
+    ])
+  })
+
+  it('sets repository URL if host is provided', async () => {
+    const log = [
+      '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}',
+      '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}'
+    ].join('\n')
+    const result = extractProgress(log, {
+      repositoryBaseUrl: 'https://gitlab.example.com'
+    })
+    assert.deepStrictEqual(result, [
+      {
+        repository: 'foo/bar',
+        state: 'started',
+        repositoryUrl: 'https://gitlab.example.com/foo/bar'
+      },
+      {
+        repository: 'foo/baz/qux',
+        state: 'pending',
+        repositoryUrl: 'https://gitlab.example.com/foo/baz/qux'
+      },
+      {
+        repository: 'random',
+        state: 'pending',
+        repositoryUrl: 'https://gitlab.example.com/random'
+      }
+    ])
+  })
+})