From 43c3bb8f059748f89bfc23c968815bdccfd0c7dc Mon Sep 17 00:00:00 2001 From: gferraro Date: Tue, 10 Dec 2024 20:13:10 +1300 Subject: [PATCH 1/5] add retry reprocessed endpoint --- api/api/V1/Reprocess.ts | 39 ++++++++++++- api/classifications/classifications.js | 58 +++++++++---------- api/models/Recording.ts | 12 +++- browse/src/api/Recording.api.ts | 6 +- .../src/components/Video/VideoRecording.vue | 2 +- 5 files changed, 80 insertions(+), 37 deletions(-) diff --git a/api/api/V1/Reprocess.ts b/api/api/V1/Reprocess.ts index b80dc9c8..be4e7800 100755 --- a/api/api/V1/Reprocess.ts +++ b/api/api/V1/Reprocess.ts @@ -28,7 +28,7 @@ import { import { idOf } from "../validation-middleware.js"; import { successResponse } from "./responseUtil.js"; import type { NextFunction } from "express-serve-static-core"; -import { ClientError } from "../customErrors.js"; +import { ClientError, BadRequestError } from "../customErrors.js"; import { arrayOf, jsonSchemaOf } from "../schema-validation.js"; import lodash from "lodash"; import RecordingIdSchema from "@schemas/api/common/RecordingId.schema.json" assert { type: "json" }; @@ -38,6 +38,43 @@ const { uniq: dedupe } = lodash; export default (app: Application, baseUrl: string) => { const apiUrl = `${baseUrl}/reprocess`; + /** + * @api {get} /api/v1/reprocess/retry/:id Retry processing a single recording which is in a failed state + * @apiName Reprocess + * @apiGroup Recordings + * @apiParam {Integer} id of recording to retry + * @apiDescription Retries processing a recording thats in a failed state + * + * @apiUse V1UserAuthorizationHeader + * + * @apiUse V1ResponseSuccess + * @apiUse V1ResponseError + */ + app.get( + `${apiUrl}/:id`, + extractJwtAuthorizedUser, + validateFields([idOf(param("id"))]), + fetchAuthorizedRequiredRecordingById(param("id")), + async (request: Request, response: Response, next) => { + if (!response.locals.recordings.isFailed()) { + return next( + new BadRequestError( + `Recording is not in a failed state '${response.locals.recordings.processingState}'` + ) + ); + } + if (await response.locals.recording.retryProcessing()) { + return successResponse(response, "Recording reprocessed"); + } else { + return next( + new BadRequestError( + `Could not retry processing of recordings ${response.locals.recordings.id}` + ) + ); + } + } + ); + /** * @api {get} /api/v1/reprocess/:id Reprocess a single recording * @apiName Reprocess diff --git a/api/classifications/classifications.js b/api/classifications/classifications.js index 8dccfbf8..75904228 100644 --- a/api/classifications/classifications.js +++ b/api/classifications/classifications.js @@ -1,37 +1,33 @@ import Classifications from "@/classifications/classification.json" assert { type: "json" }; const flattenNodes = (acc, node, parentPath) => { - for (const child of node.children || []) { - acc[child.label] = { - label: child.label, - display: child.display || child.label, - path: `${parentPath}.${child.label}`, - }; - flattenNodes(acc, child, acc[child.label].path); - } - return acc; + for (const child of node.children || []) { + acc[child.label] = { + label: child.label, + display: child.display || child.label, + path: `${parentPath}.${child.label}`, + }; + flattenNodes(acc, child, acc[child.label].path); + } + return acc; }; export const flatClassifications = (() => { - const nodes = flattenNodes({}, Classifications, "all"); - if (nodes.unknown) { - nodes["unidentified"] = nodes["unknown"]; - } - return nodes; + const nodes = flattenNodes({}, Classifications, "all"); + if (nodes.unknown) { + nodes["unidentified"] = nodes["unknown"]; + } + return nodes; })(); -export const displayLabelForClassificationLabel = ( - label, - aiTag = false, - isAudioContext = false -) => { - label = label.toLowerCase(); - if (label === "unclassified") { - return "AI Queued"; - } - if (label === "unidentified" && aiTag) { - return "Unidentified"; - } - const classifications = flatClassifications; - if ((label === "human" || label === "person") && !isAudioContext) { - return "human"; - } - return (classifications[label] && classifications[label].display) || label; +export const displayLabelForClassificationLabel = (label, aiTag = false, isAudioContext = false) => { + label = label.toLowerCase(); + if (label === "unclassified") { + return "AI Queued"; + } + if (label === "unidentified" && aiTag) { + return "Unidentified"; + } + const classifications = flatClassifications; + if ((label === "human" || label === "person") && !isAudioContext) { + return "human"; + } + return (classifications[label] && classifications[label].display) || label; }; diff --git a/api/models/Recording.ts b/api/models/Recording.ts index 02edbbc4..f6ed6d3e 100755 --- a/api/models/Recording.ts +++ b/api/models/Recording.ts @@ -209,7 +209,7 @@ export interface Recording extends Sequelize.Model, ModelCommon { getGroup: () => Promise; getActiveTracksTagsAndTagger: () => Promise; - + retryProcessing: () => Promise; reprocess: () => Promise; filterData: (options: any) => void; // NOTE: Implicitly created by sequelize associations (along with other @@ -787,6 +787,16 @@ from ( }; } + // retry processing this recording + Recording.prototype.retryProcessing = async function () { + if (!this.processingState.endsWith(".failed")) { + return null; + } + await this.update({ + processingState: this.processingState.replace(".failed", ""), + }); + }; + // reprocess a recording and set all active tracks to archived Recording.prototype.reprocess = async function () { const tags = await this.getTags(); diff --git a/browse/src/api/Recording.api.ts b/browse/src/api/Recording.api.ts index d44cbc67..cc1ae1db 100644 --- a/browse/src/api/Recording.api.ts +++ b/browse/src/api/Recording.api.ts @@ -546,8 +546,8 @@ function deleteRecordingTag( return CacophonyApi.delete(`${apiPath}/${id}/tags/${tagId}`); } -function reprocess(id: RecordingId): Promise> { - return CacophonyApi.get(`/api/v1/reprocess/${id}`); +function retryProcessing(id: RecordingId): Promise> { + return CacophonyApi.get(`/api/v1/reprocess/retry/${id}`); } function thumbnail(id: RecordingId): string { @@ -641,7 +641,7 @@ export default { deleteTrack, undeleteTrack, updateTrack, - reprocess, + retryProcessing, addTrackTag, deleteTrackTag, replaceTrackTag, diff --git a/browse/src/components/Video/VideoRecording.vue b/browse/src/components/Video/VideoRecording.vue index bc5777c5..f3262ca3 100644 --- a/browse/src/components/Video/VideoRecording.vue +++ b/browse/src/components/Video/VideoRecording.vue @@ -329,7 +329,7 @@ export default { }, methods: { async reprocess() { - const { success } = await api.recording.reprocess(this.recordingId); + const { success } = await api.recording.retryProcessing(this.recordingId); if (success) { this.$emit("recording-updated", { id: this.recordingId, From 3c5ab4d92fde2a6c87898bdc5e20820190af0f17 Mon Sep 17 00:00:00 2001 From: gferraro Date: Wed, 11 Dec 2024 08:36:27 +1300 Subject: [PATCH 2/5] change name and tidy up --- api/api/V1/Reprocess.ts | 15 ++++++++------- api/models/Recording.ts | 7 ++++--- browse/src/api/Recording.api.ts | 6 +++--- browse/src/components/Video/VideoRecording.vue | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/api/api/V1/Reprocess.ts b/api/api/V1/Reprocess.ts index be4e7800..1bde6856 100755 --- a/api/api/V1/Reprocess.ts +++ b/api/api/V1/Reprocess.ts @@ -39,7 +39,7 @@ export default (app: Application, baseUrl: string) => { const apiUrl = `${baseUrl}/reprocess`; /** - * @api {get} /api/v1/reprocess/retry/:id Retry processing a single recording which is in a failed state + * @api {get} /api/v1/reprocess/retryFailed/:id Retry processing a single recording which is in a failed state * @apiName Reprocess * @apiGroup Recordings * @apiParam {Integer} id of recording to retry @@ -51,24 +51,24 @@ export default (app: Application, baseUrl: string) => { * @apiUse V1ResponseError */ app.get( - `${apiUrl}/:id`, + `${apiUrl}/retryFailed/:id`, extractJwtAuthorizedUser, validateFields([idOf(param("id"))]), fetchAuthorizedRequiredRecordingById(param("id")), async (request: Request, response: Response, next) => { - if (!response.locals.recordings.isFailed()) { + if (!response.locals.recording.isFailed()) { return next( new BadRequestError( - `Recording is not in a failed state '${response.locals.recordings.processingState}'` + `Recording is not in a failed state '${response.locals.recording.processingState}'` ) ); } - if (await response.locals.recording.retryProcessing()) { + if (await response.locals.recording.retryFailed()) { return successResponse(response, "Recording reprocessed"); } else { return next( new BadRequestError( - `Could not retry processing of recordings ${response.locals.recordings.id}` + `Could not retry processing of recordings ${response.locals.recording.id}` ) ); } @@ -80,7 +80,8 @@ export default (app: Application, baseUrl: string) => { * @apiName Reprocess * @apiGroup Recordings * @apiParam {Integer} id of recording to reprocess - * @apiDescription Marks a recording for reprocessing and archives existing tracks + * @apiDescription Marks a recording for reprocessing (tracking), and archives existing tracks. + * Used if tracking algorithms have changed * * @apiUse V1UserAuthorizationHeader * diff --git a/api/models/Recording.ts b/api/models/Recording.ts index f6ed6d3e..b0c25560 100755 --- a/api/models/Recording.ts +++ b/api/models/Recording.ts @@ -209,7 +209,7 @@ export interface Recording extends Sequelize.Model, ModelCommon { getGroup: () => Promise; getActiveTracksTagsAndTagger: () => Promise; - retryProcessing: () => Promise; + retryFailed: () => Promise; reprocess: () => Promise; filterData: (options: any) => void; // NOTE: Implicitly created by sequelize associations (along with other @@ -788,13 +788,14 @@ from ( } // retry processing this recording - Recording.prototype.retryProcessing = async function () { + Recording.prototype.retryFailed = async function () { if (!this.processingState.endsWith(".failed")) { - return null; + return false; } await this.update({ processingState: this.processingState.replace(".failed", ""), }); + return true; }; // reprocess a recording and set all active tracks to archived diff --git a/browse/src/api/Recording.api.ts b/browse/src/api/Recording.api.ts index cc1ae1db..e3aedc56 100644 --- a/browse/src/api/Recording.api.ts +++ b/browse/src/api/Recording.api.ts @@ -546,8 +546,8 @@ function deleteRecordingTag( return CacophonyApi.delete(`${apiPath}/${id}/tags/${tagId}`); } -function retryProcessing(id: RecordingId): Promise> { - return CacophonyApi.get(`/api/v1/reprocess/retry/${id}`); +function retryFailed(id: RecordingId): Promise> { + return CacophonyApi.get(`/api/v1/reprocess/retryFailed/${id}`); } function thumbnail(id: RecordingId): string { @@ -641,7 +641,7 @@ export default { deleteTrack, undeleteTrack, updateTrack, - retryProcessing, + retryFailed, addTrackTag, deleteTrackTag, replaceTrackTag, diff --git a/browse/src/components/Video/VideoRecording.vue b/browse/src/components/Video/VideoRecording.vue index f3262ca3..e49a3ade 100644 --- a/browse/src/components/Video/VideoRecording.vue +++ b/browse/src/components/Video/VideoRecording.vue @@ -329,7 +329,7 @@ export default { }, methods: { async reprocess() { - const { success } = await api.recording.retryProcessing(this.recordingId); + const { success } = await api.recording.retryFailed(this.recordingId); if (success) { this.$emit("recording-updated", { id: this.recordingId, From 276092fa072e5bca52c371ecd7565a388336f647 Mon Sep 17 00:00:00 2001 From: gferraro Date: Wed, 11 Dec 2024 08:48:51 +1300 Subject: [PATCH 3/5] fix endpoint name style --- api/api/V1/Reprocess.ts | 2 +- browse/src/api/Recording.api.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/api/V1/Reprocess.ts b/api/api/V1/Reprocess.ts index 1bde6856..04d8e3b0 100755 --- a/api/api/V1/Reprocess.ts +++ b/api/api/V1/Reprocess.ts @@ -51,7 +51,7 @@ export default (app: Application, baseUrl: string) => { * @apiUse V1ResponseError */ app.get( - `${apiUrl}/retryFailed/:id`, + `${apiUrl}/retry-failed/:id`, extractJwtAuthorizedUser, validateFields([idOf(param("id"))]), fetchAuthorizedRequiredRecordingById(param("id")), diff --git a/browse/src/api/Recording.api.ts b/browse/src/api/Recording.api.ts index e3aedc56..b90836f8 100644 --- a/browse/src/api/Recording.api.ts +++ b/browse/src/api/Recording.api.ts @@ -547,7 +547,7 @@ function deleteRecordingTag( } function retryFailed(id: RecordingId): Promise> { - return CacophonyApi.get(`/api/v1/reprocess/retryFailed/${id}`); + return CacophonyApi.get(`/api/v1/reprocess/retry-failed/${id}`); } function thumbnail(id: RecordingId): string { From 2b38a15b8f4aa52ac2c87e0826af79151b53924c Mon Sep 17 00:00:00 2001 From: gferraro Date: Wed, 11 Dec 2024 08:50:20 +1300 Subject: [PATCH 4/5] doc string --- api/api/V1/Reprocess.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api/V1/Reprocess.ts b/api/api/V1/Reprocess.ts index 04d8e3b0..f829a588 100755 --- a/api/api/V1/Reprocess.ts +++ b/api/api/V1/Reprocess.ts @@ -39,7 +39,7 @@ export default (app: Application, baseUrl: string) => { const apiUrl = `${baseUrl}/reprocess`; /** - * @api {get} /api/v1/reprocess/retryFailed/:id Retry processing a single recording which is in a failed state + * @api {get} /api/v1/reprocess/retry-failed/:id Retry processing a single recording which is in a failed state * @apiName Reprocess * @apiGroup Recordings * @apiParam {Integer} id of recording to retry From 3d81ca7410acd9128664af9c0f28fe6f0ec69e1b Mon Sep 17 00:00:00 2001 From: gferraro Date: Wed, 11 Dec 2024 09:11:38 +1300 Subject: [PATCH 5/5] only check first processing time for wait time query --- api/scripts/influx-metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/scripts/influx-metrics.ts b/api/scripts/influx-metrics.ts index dd76be2f..6a1e9d5c 100755 --- a/api/scripts/influx-metrics.ts +++ b/api/scripts/influx-metrics.ts @@ -62,7 +62,7 @@ async function measureProcessingWaitTime(influx, pgClient) { const res = await pgQuery( pgClient, `select "createdAt" from "Recordings" - where "processingState" in ('analyse', 'tracking') and "deletedAt" is null + where "processingState" in ('analyse', 'tracking') and "deletedAt" is null and "processingFailedCount" = 0 order by "createdAt" asc limit 1` );