From 060f041d0606d5ac1b4a2ead99f4c57b0fa1b3c7 Mon Sep 17 00:00:00 2001 From: Fabian Meyer <3982806+meyfa@users.noreply.github.com> Date: Sat, 13 Jul 2024 12:39:32 +0200 Subject: [PATCH] fix: Refactor progress extraction and add tests (#40) --- backend/src/api/pod-progress.ts | 77 +--------------- backend/src/renovate/progress.ts | 75 +++++++++++++++ backend/test/renovate/progress.test.ts | 122 +++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 75 deletions(-) create mode 100644 backend/src/renovate/progress.ts create mode 100644 backend/test/renovate/progress.test.ts diff --git a/backend/src/api/pod-progress.ts b/backend/src/api/pod-progress.ts index 274db8a..db9a3a0 100644 --- a/backend/src/api/pod-progress.ts +++ b/backend/src/api/pod-progress.ts @@ -2,16 +2,9 @@ import { FastifyPluginAsync } from 'fastify' import { Controllers } from '../controllers.js' import { forbidden, notFound } from './errors.js' import { authenticateSession } from '../auth/common.js' -import sjson from 'secure-json-parse' +import { extractProgress, ProgressItem } from '../renovate/progress.js' import { BackendConfig } from '../backend-config.js' -interface ProgressItem { - repository: string - repositoryUrl?: string - state: 'pending' | 'started' | 'finished' - duration?: number -} - export interface PodProgressRoute { Reply: ProgressItem[] } @@ -32,7 +25,7 @@ export const podProgressRoute = ({ logsController }: Controllers, config: Backen if (logs == null) { return await notFound(reply) } - const progress = getProgress(logs, config) + const progress = extractProgress(logs, { repositoryBaseUrl: config.gitlab.host }) if (progress == null) { // In case the log cannot be parsed, progress is unavailable, so return a 404. return await notFound((reply)) @@ -40,69 +33,3 @@ export const podProgressRoute = ({ logsController }: Controllers, config: Backen return progress }) } - -function getProgress (logs: string, config: BackendConfig): ProgressItem[] | undefined { - const repositories = new Map() - let foundRepositories = false - for (const line of lines(logs)) { - const message = tryParseLogMessage(line) - // Note: While it looks ugly, manual validation is more than 3x faster than using superstruct. - // This is worth it here as there is lots of data to process. - if (message == null || typeof message !== 'object' || !('msg' in message)) { - continue - } - // The initial log message is for autodiscovery and tells us about the repositories that will be processed. - if ('repositories' in message && Array.isArray(message.repositories) && - message.repositories.every(msg => typeof msg === 'string')) { - foundRepositories = true - for (const repository of message.repositories) { - const item: ProgressItem = { repository, state: 'pending' } - if (config.gitlab.host != null) { - item.repositoryUrl = new URL(repository, config.gitlab.host).toString() - } - repositories.set(repository, item) - } - } - // Log messages associated with a repository have a repository property, and update the item's status. - if (!('repository' in message) || typeof message.repository !== 'string') { - continue - } - const item = repositories.get(message.repository) - if (item == null) { - continue // what? - } - switch (message.msg) { - case 'Repository started': - item.state = 'started' - break - case 'Repository finished': - item.state = 'finished' - break - } - } - if (!foundRepositories) { - return undefined - } - // Note: JS Map always returns values in insertion order. - return Array.from(repositories.values()) -} - -function * lines (str: string): Generator { - let lineFrom = 0 - while (lineFrom < str.length) { - let lineTo = str.indexOf('\n', lineFrom) - if (lineTo === -1) { - lineTo = str.length - } - yield str.slice(lineFrom, lineTo) - lineFrom = lineTo + 1 - } -} - -function tryParseLogMessage (str: string): unknown { - try { - return sjson.parse(str) - } catch { - return undefined - } -} diff --git a/backend/src/renovate/progress.ts b/backend/src/renovate/progress.ts new file mode 100644 index 0000000..0e6287f --- /dev/null +++ b/backend/src/renovate/progress.ts @@ -0,0 +1,75 @@ +import sjson from 'secure-json-parse' + +export interface ProgressItem { + repository: string + repositoryUrl?: string + state: 'pending' | 'started' | 'finished' +} + +export function extractProgress (logs: string, options: { + repositoryBaseUrl?: string +}): ProgressItem[] | undefined { + const repositories = new Map() + let foundRepositories = false + for (const line of lines(logs)) { + const message = tryParseLogMessage(line) + // Note: While it looks ugly, manual validation is more than 3x faster than using superstruct. + // This is worth it here as there is lots of data to process. + if (message == null || typeof message !== 'object' || !('msg' in message)) { + continue + } + // The initial log message is for autodiscovery and tells us about the repositories that will be processed. + if ('repositories' in message && Array.isArray(message.repositories) && + message.repositories.every(msg => typeof msg === 'string')) { + foundRepositories = true + for (const repository of message.repositories) { + const item: ProgressItem = { repository, state: 'pending' } + if (options.repositoryBaseUrl != null) { + item.repositoryUrl = new URL(repository, options.repositoryBaseUrl).toString() + } + repositories.set(repository, item) + } + } + // Log messages associated with a repository have a repository property, and update the item's status. + if (!('repository' in message) || typeof message.repository !== 'string') { + continue + } + const item = repositories.get(message.repository) + if (item == null) { + continue // what? + } + switch (message.msg) { + case 'Repository started': + item.state = 'started' + break + case 'Repository finished': + item.state = 'finished' + break + } + } + if (!foundRepositories) { + return undefined + } + // Note: JS Map always returns values in insertion order. + return Array.from(repositories.values()) +} + +function * lines (str: string): Generator { + let lineFrom = 0 + while (lineFrom < str.length) { + let lineTo = str.indexOf('\n', lineFrom) + if (lineTo === -1) { + lineTo = str.length + } + yield str.slice(lineFrom, lineTo) + lineFrom = lineTo + 1 + } +} + +function tryParseLogMessage (str: string): unknown { + try { + return sjson.parse(str) + } catch { + return undefined + } +} diff --git a/backend/test/renovate/progress.test.ts b/backend/test/renovate/progress.test.ts new file mode 100644 index 0000000..817dce8 --- /dev/null +++ b/backend/test/renovate/progress.test.ts @@ -0,0 +1,122 @@ +import assert from 'node:assert' +import { extractProgress } from '../../src/renovate/progress.js' + +describe('renovate/progress.ts', () => { + describe('extractProgress()', () => { + it('returns undefined for empty input', async () => { + const result = extractProgress('', {}) + assert.strictEqual(result, undefined) + }) + + it('ignores non-JSON input', async () => { + const result = extractProgress('foo bar\nbaz qux 42 {}', {}) + assert.strictEqual(result, undefined) + }) + + it('detects repository list', async () => { + const log = [ + '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}' + ].join('\n') + const result = extractProgress(log, {}) + assert.deepStrictEqual(result, [ + { + repository: 'foo/bar', + state: 'pending' + }, + { + repository: 'foo/baz/qux', + state: 'pending' + }, + { + repository: 'random', + state: 'pending' + } + ]) + }) + + it('detects repository started/finished', async () => { + const log = [ + '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","cloned":true,"durationMs":7133,"msg":"Repository finished","time":"2024-07-12T17:00:30.123Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/baz/qux","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:32.456Z","v":0}' + ].join('\n') + const result = extractProgress(log, {}) + assert.deepStrictEqual(result, [ + { + repository: 'foo/bar', + state: 'finished' + }, + { + repository: 'foo/baz/qux', + state: 'started' + }, + { + repository: 'random', + state: 'pending' + } + ]) + }) + }) + + it('ignores invalid log lines', async () => { + const log = [ + '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}', + 'null', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}', + 'false', + 'true', + '{}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}', + '[{"msg:"something"}]', + 'some random text', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","cloned":true,"durationMs":7133,"msg":"Repository finished","time":"2024-07-12T17:00:30.123Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/baz/qux","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:32.456Z","v":0}' + ].join('\n') + const result = extractProgress(log, {}) + assert.deepStrictEqual(result, [ + { + repository: 'foo/bar', + state: 'finished' + }, + { + repository: 'foo/baz/qux', + state: 'started' + }, + { + repository: 'random', + state: 'pending' + } + ]) + }) + + it('sets repository URL if host is provided', async () => { + const log = [ + '{"name":"renovate","hostname":"renovate-foo-bar","pid":10,"level":30,"logContext":"abcd","msg":"test message","time":"2024-07-12T17:00:06.051Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","length":3,"repositories":["foo/bar", "foo/baz/qux", "random"],"msg":"Autodiscovered repositories","time":"2024-07-12T17:00:08.848Z","v":0}', + '{"name":"renovate","hostname":"renovate-1337","pid":10,"level":30,"logContext":"abcd","repository":"foo/bar","renovateVersion":"12.345.6","msg":"Repository started","time":"2024-07-12T17:00:20.861Z","v":0}' + ].join('\n') + const result = extractProgress(log, { + repositoryBaseUrl: 'https://gitlab.example.com' + }) + assert.deepStrictEqual(result, [ + { + repository: 'foo/bar', + state: 'started', + repositoryUrl: 'https://gitlab.example.com/foo/bar' + }, + { + repository: 'foo/baz/qux', + state: 'pending', + repositoryUrl: 'https://gitlab.example.com/foo/baz/qux' + }, + { + repository: 'random', + state: 'pending', + repositoryUrl: 'https://gitlab.example.com/random' + } + ]) + }) +})