From addc1c4bfb13fc168e8aabd688af4f11eeed9940 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 May 2024 20:31:32 +0000 Subject: [PATCH 01/11] chore(release): 0.7.0-beta.1 [skip ci] ## [0.7.0-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0-beta.1) (2024-05-07) ### Features * add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a)) * update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f)) * upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51)) ### Bug Fixes * get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb)) --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85a902a1..18341bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## [0.7.0-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0-beta.1) (2024-05-07) + + +### Features + +* add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a)) +* update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f)) +* upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51)) + + +### Bug Fixes + +* get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb)) + ## [0.6.3](https://github.com/hirosystems/token-metadata-api/compare/v0.6.2...v0.6.3) (2024-05-07) From 5e1af5c28cd0b1313f78a59b015669ceb07e5738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Tue, 7 May 2024 15:59:34 -0600 Subject: [PATCH 02/11] fix: reuse gcs token and validate image cache script errors (#213) --- config/image-cache.js | 35 ++++++++++++++++++------- src/token-processor/util/image-cache.ts | 9 +++---- tests/image-cache.test.ts | 8 ++++-- tests/test-image-cache-error.js | 3 +++ 4 files changed, 38 insertions(+), 17 deletions(-) create mode 100755 tests/test-image-cache-error.js diff --git a/config/image-cache.js b/config/image-cache.js index 925863aa..1377c224 100755 --- a/config/image-cache.js +++ b/config/image-cache.js @@ -46,7 +46,11 @@ async function getGcsAuthToken() { headers: { 'Metadata-Flavor': 'Google' }, } ); - if (response.data?.access_token) return response.data.access_token; + if (response.data?.access_token) { + // Cache the token so we can reuse it for other images. + process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = response.data.access_token; + return response.data.access_token; + } throw new Error(`GCS token not found`); } catch (error) { throw new Error(`Error fetching GCS access token: ${error.message}`); @@ -99,15 +103,26 @@ fetch( passThrough.pipe(fullSizeTransform); passThrough.pipe(thumbnailTransform); - const authToken = await getGcsAuthToken(); - const results = await Promise.all([ - upload(fullSizeTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}.png`, authToken), - upload(thumbnailTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}-thumb.png`, authToken), - ]); - - // The API will read these strings as CDN URLs. - for (const result of results) console.log(result); + let didRetryUnauthorized = false; + while (true) { + const authToken = await getGcsAuthToken(); + try { + const results = await Promise.all([ + upload(fullSizeTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}.png`, authToken), + upload(thumbnailTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}-thumb.png`, authToken), + ]); + // The API will read these strings as CDN URLs. + for (const result of results) console.log(result); + break; + } catch (error) { + if ((error.message.endsWith('403') || error.message.endsWith('401')) && !didRetryUnauthorized) { + // Force a dynamic token refresh and try again. + process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = undefined; + didRetryUnauthorized = true; + } else throw error; + } + } }) .catch(error => { - throw new Error(`Error processing image: ${error.message}`); + throw new Error(`Error fetching image: ${error}`); }); diff --git a/src/token-processor/util/image-cache.ts b/src/token-processor/util/image-cache.ts index 57eac663..66529a49 100644 --- a/src/token-processor/util/image-cache.ts +++ b/src/token-processor/util/image-cache.ts @@ -24,13 +24,14 @@ export async function processImageUrl( if (imgUrl.startsWith('data:')) { return [imgUrl]; } + logger.info(`ImageCache processing image for token ${contractPrincipal} (${tokenNumber})...`); const repoDir = process.cwd(); const { code, stdout, stderr } = await new Promise<{ code: number; stdout: string; stderr: string; - }>((resolve, reject) => { + }>(resolve => { const cp = child_process.spawn( imageCacheProcessor, [imgUrl, contractPrincipal, tokenNumber.toString()], @@ -41,11 +42,9 @@ export async function processImageUrl( cp.stdout.on('data', data => (stdout += data)); cp.stderr.on('data', data => (stderr += data)); cp.on('close', code => resolve({ code: code ?? 0, stdout, stderr })); - cp.on('error', error => reject(error)); }); - if (code !== 0 && stderr) { - logger.warn(stderr, `ImageCache error`); - } + if (code !== 0) throw new Error(`ImageCache error: ${stderr}`); + const result = stdout.trim().split('\n'); try { return result.map(r => new URL(r).toString()); diff --git a/tests/image-cache.test.ts b/tests/image-cache.test.ts index 13baa5e9..747774bc 100644 --- a/tests/image-cache.test.ts +++ b/tests/image-cache.test.ts @@ -4,13 +4,13 @@ import { processImageUrl } from '../src/token-processor/util/image-cache'; describe('Image cache', () => { const contract = 'SP3QSAJQ4EA8WXEDSRRKMZZ29NH91VZ6C5X88FGZQ.crashpunks-v2'; const tokenNumber = 100n; + const url = 'http://cloudflare-ipfs.com/test/image.png'; beforeAll(() => { ENV.METADATA_IMAGE_CACHE_PROCESSOR = './tests/test-image-cache.js'; }); test('transforms image URL correctly', async () => { - const url = 'http://cloudflare-ipfs.com/test/image.png'; const transformed = await processImageUrl(url, contract, tokenNumber); expect(transformed).toStrictEqual([ 'http://cloudflare-ipfs.com/test/image.png?processed=true', @@ -26,8 +26,12 @@ describe('Image cache', () => { test('ignores empty script paths', async () => { ENV.METADATA_IMAGE_CACHE_PROCESSOR = ''; - const url = 'http://cloudflare-ipfs.com/test/image.png'; const transformed = await processImageUrl(url, contract, tokenNumber); expect(transformed).toStrictEqual([url]); }); + + test('handles script errors', async () => { + ENV.METADATA_IMAGE_CACHE_PROCESSOR = './tests/test-image-cache-error.js'; + await expect(processImageUrl(url, contract, tokenNumber)).rejects.toThrow(/ImageCache error/); + }); }); diff --git a/tests/test-image-cache-error.js b/tests/test-image-cache-error.js new file mode 100755 index 00000000..4ab0feb6 --- /dev/null +++ b/tests/test-image-cache-error.js @@ -0,0 +1,3 @@ +#!/usr/bin/env node +console.error('Test error'); +throw new Error('Test'); From 2f7d2cc5da44907140995928c803b7adc13addc9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 May 2024 22:01:47 +0000 Subject: [PATCH 03/11] chore(release): 0.7.0-beta.2 [skip ci] ## [0.7.0-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.1...v0.7.0-beta.2) (2024-05-07) ### Bug Fixes * reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18341bf2..119c1d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.7.0-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.1...v0.7.0-beta.2) (2024-05-07) + + +### Bug Fixes + +* reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738)) + ## [0.7.0-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0-beta.1) (2024-05-07) From c74819cb972111ab22681cbd66b7f25efe54d9d9 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 7 May 2024 17:30:58 -0600 Subject: [PATCH 04/11] chore: improve logging for imagecache --- src/token-processor/util/image-cache.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/token-processor/util/image-cache.ts b/src/token-processor/util/image-cache.ts index 66529a49..9ad70e42 100644 --- a/src/token-processor/util/image-cache.ts +++ b/src/token-processor/util/image-cache.ts @@ -25,7 +25,7 @@ export async function processImageUrl( return [imgUrl]; } - logger.info(`ImageCache processing image for token ${contractPrincipal} (${tokenNumber})...`); + logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`); const repoDir = process.cwd(); const { code, stdout, stderr } = await new Promise<{ code: number; @@ -44,8 +44,9 @@ export async function processImageUrl( cp.on('close', code => resolve({ code: code ?? 0, stdout, stderr })); }); if (code !== 0) throw new Error(`ImageCache error: ${stderr}`); - const result = stdout.trim().split('\n'); + logger.info(result, `ImageCache processed token ${contractPrincipal} (${tokenNumber})`); + try { return result.map(r => new URL(r).toString()); } catch (error) { From 5826628a329225fbf697a092dc201fc74fb96d43 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 7 May 2024 17:37:02 -0600 Subject: [PATCH 05/11] fix: image cache agent arg types --- config/image-cache.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/image-cache.js b/config/image-cache.js index 1377c224..181e65ff 100755 --- a/config/image-cache.js +++ b/config/image-cache.js @@ -34,6 +34,9 @@ const IMAGE_RESIZE_WIDTH = parseInt(process.env['IMAGE_CACHE_RESIZE_WIDTH'] ?? ' const GCS_BUCKET_NAME = process.env['IMAGE_CACHE_GCS_BUCKET_NAME']; const GCS_OBJECT_NAME_PREFIX = process.env['IMAGE_CACHE_GCS_OBJECT_NAME_PREFIX']; const CDN_BASE_PATH = process.env['IMAGE_CACHE_CDN_BASE_PATH']; +const TIMEOUT = parseInt(process.env['METADATA_FETCH_TIMEOUT_MS'] ?? '30'); +const MAX_REDIRECTIONS = parseInt(process.env['METADATA_FETCH_MAX_REDIRECTIONS'] ?? '0'); +const MAX_RESPONSE_SIZE = parseInt(process.env['IMAGE_CACHE_MAX_BYTE_SIZE'] ?? '-1'); async function getGcsAuthToken() { const envToken = process.env['IMAGE_CACHE_GCS_AUTH_TOKEN']; @@ -78,10 +81,10 @@ fetch( IMAGE_URL, { dispatcher: new Agent({ - headersTimeout: process.env['METADATA_FETCH_TIMEOUT_MS'], - bodyTimeout: process.env['METADATA_FETCH_TIMEOUT_MS'], - maxRedirections: process.env['METADATA_FETCH_MAX_REDIRECTIONS'], - maxResponseSize: process.env['IMAGE_CACHE_MAX_BYTE_SIZE'], + headersTimeout: TIMEOUT, + bodyTimeout: TIMEOUT, + maxRedirections: MAX_REDIRECTIONS, + maxResponseSize: MAX_RESPONSE_SIZE, connect: { rejectUnauthorized: false, // Ignore SSL cert errors. }, From 6a51673d964ea85a0a5a779b02ae222d342b0652 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 May 2024 23:39:06 +0000 Subject: [PATCH 06/11] chore(release): 0.7.0-beta.3 [skip ci] ## [0.7.0-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.2...v0.7.0-beta.3) (2024-05-07) ### Bug Fixes * image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 119c1d65..91d01193 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.7.0-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.2...v0.7.0-beta.3) (2024-05-07) + + +### Bug Fixes + +* image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43)) + ## [0.7.0-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.1...v0.7.0-beta.2) (2024-05-07) From a6b98c5099a9de1d88e74eed66dece1c4c157422 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 7 May 2024 22:33:13 -0600 Subject: [PATCH 07/11] fix: get access token properly --- config/image-cache.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/config/image-cache.js b/config/image-cache.js index 181e65ff..6497e885 100755 --- a/config/image-cache.js +++ b/config/image-cache.js @@ -49,12 +49,13 @@ async function getGcsAuthToken() { headers: { 'Metadata-Flavor': 'Google' }, } ); - if (response.data?.access_token) { + const json = await response.body.json(); + if (response.statusCode === 200 && json.access_token) { // Cache the token so we can reuse it for other images. - process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = response.data.access_token; - return response.data.access_token; + process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = json.access_token; + return json.access_token; } - throw new Error(`GCS token not found`); + throw new Error(`GCS access token not found ${response.statusCode}: ${json}`); } catch (error) { throw new Error(`Error fetching GCS access token: ${error.message}`); } @@ -118,7 +119,10 @@ fetch( for (const result of results) console.log(result); break; } catch (error) { - if ((error.message.endsWith('403') || error.message.endsWith('401')) && !didRetryUnauthorized) { + if ( + (error.message.endsWith('403') || error.message.endsWith('401')) && + !didRetryUnauthorized + ) { // Force a dynamic token refresh and try again. process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = undefined; didRetryUnauthorized = true; From a3aad1d990e1dacb896cf372d78dedb6a712292a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 8 May 2024 04:35:56 +0000 Subject: [PATCH 08/11] chore(release): 0.7.0-beta.4 [skip ci] ## [0.7.0-beta.4](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.3...v0.7.0-beta.4) (2024-05-08) ### Bug Fixes * get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91d01193..59ac9ecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.7.0-beta.4](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.3...v0.7.0-beta.4) (2024-05-08) + + +### Bug Fixes + +* get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422)) + ## [0.7.0-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.2...v0.7.0-beta.3) (2024-05-07) From 115a745c268e7bb8115a488ca111e8b46cefed62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Mon, 13 May 2024 10:33:40 -0600 Subject: [PATCH 09/11] fix: improve image cache error handling (#214) * fix: handle cache script errors * fix: tests * fix: try fetch in a function * fix: force timeouts to test error handling * fix: use error codes in script * fix: unauthorized error handling * fix: return to old fetch type * fix: handle exit code explicitly * chore: try again with code exit * fix: take code from error cause * fix: generate retryable errors * fix: remove explicit code zero * chore: debug log error * chore: debug without all * chore: debug without all 2 * chore: debug more * fix: prmise await * chore: detailed logs * fix: log response * fix: logger * fix: try with only one handler * fix: add error handlers to streams * fix: mark invalid out as retryable * fix: validate if a cause exists * fix: retry ECONNRESET * chore: comment --- config/image-cache.js | 76 ++++++------ .../queue/job/process-token-job.ts | 4 +- src/token-processor/util/image-cache.ts | 108 +++++++++++++----- src/token-processor/util/metadata-helpers.ts | 14 +-- tests/image-cache.test.ts | 12 +- tests/metadata-helpers.test.ts | 8 +- tests/test-image-cache-error.js | 2 +- 7 files changed, 142 insertions(+), 82 deletions(-) diff --git a/config/image-cache.js b/config/image-cache.js index 6497e885..d9d6f100 100755 --- a/config/image-cache.js +++ b/config/image-cache.js @@ -34,7 +34,7 @@ const IMAGE_RESIZE_WIDTH = parseInt(process.env['IMAGE_CACHE_RESIZE_WIDTH'] ?? ' const GCS_BUCKET_NAME = process.env['IMAGE_CACHE_GCS_BUCKET_NAME']; const GCS_OBJECT_NAME_PREFIX = process.env['IMAGE_CACHE_GCS_OBJECT_NAME_PREFIX']; const CDN_BASE_PATH = process.env['IMAGE_CACHE_CDN_BASE_PATH']; -const TIMEOUT = parseInt(process.env['METADATA_FETCH_TIMEOUT_MS'] ?? '30'); +const TIMEOUT = parseInt(process.env['METADATA_FETCH_TIMEOUT_MS'] ?? '30000'); const MAX_REDIRECTIONS = parseInt(process.env['METADATA_FETCH_MAX_REDIRECTIONS'] ?? '0'); const MAX_RESPONSE_SIZE = parseInt(process.env['IMAGE_CACHE_MAX_BYTE_SIZE'] ?? '-1'); @@ -47,35 +47,29 @@ async function getGcsAuthToken() { { method: 'GET', headers: { 'Metadata-Flavor': 'Google' }, + throwOnError: true, } ); const json = await response.body.json(); - if (response.statusCode === 200 && json.access_token) { - // Cache the token so we can reuse it for other images. - process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = json.access_token; - return json.access_token; - } - throw new Error(`GCS access token not found ${response.statusCode}: ${json}`); + // Cache the token so we can reuse it for other images. + process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = json.access_token; + return json.access_token; } catch (error) { - throw new Error(`Error fetching GCS access token: ${error.message}`); + throw new Error(`GCS access token error: ${error}`); } } async function upload(stream, name, authToken) { - try { - const response = await request( - `https://storage.googleapis.com/upload/storage/v1/b/${GCS_BUCKET_NAME}/o?uploadType=media&name=${GCS_OBJECT_NAME_PREFIX}${name}`, - { - method: 'POST', - body: stream, - headers: { 'Content-Type': 'image/png', Authorization: `Bearer ${authToken}` }, - } - ); - if (response.statusCode !== 200) throw new Error(`GCS error: ${response.statusCode}`); - return `${CDN_BASE_PATH}${name}`; - } catch (error) { - throw new Error(`Error uploading ${name}: ${error.message}`); - } + await request( + `https://storage.googleapis.com/upload/storage/v1/b/${GCS_BUCKET_NAME}/o?uploadType=media&name=${GCS_OBJECT_NAME_PREFIX}${name}`, + { + method: 'POST', + body: stream, + headers: { 'Content-Type': 'image/png', Authorization: `Bearer ${authToken}` }, + throwOnError: true, + } + ); + return `${CDN_BASE_PATH}${name}`; } fetch( @@ -86,15 +80,13 @@ fetch( bodyTimeout: TIMEOUT, maxRedirections: MAX_REDIRECTIONS, maxResponseSize: MAX_RESPONSE_SIZE, + throwOnError: true, connect: { rejectUnauthorized: false, // Ignore SSL cert errors. }, }), }, - ({ statusCode, body }) => { - if (statusCode !== 200) throw new Error(`Failed to fetch image: ${statusCode}`); - return body; - } + ({ body }) => body ) .then(async response => { const imageReadStream = Readable.fromWeb(response.body); @@ -115,15 +107,16 @@ fetch( upload(fullSizeTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}.png`, authToken), upload(thumbnailTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}-thumb.png`, authToken), ]); - // The API will read these strings as CDN URLs. - for (const result of results) console.log(result); + for (const r of results) console.log(r); break; } catch (error) { if ( - (error.message.endsWith('403') || error.message.endsWith('401')) && - !didRetryUnauthorized + !didRetryUnauthorized && + error.cause && + error.cause.code == 'UND_ERR_RESPONSE_STATUS_CODE' && + (error.cause.statusCode === 401 || error.cause.statusCode === 403) ) { - // Force a dynamic token refresh and try again. + // GCS token is probably expired. Force a token refresh before trying again. process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = undefined; didRetryUnauthorized = true; } else throw error; @@ -131,5 +124,24 @@ fetch( } }) .catch(error => { - throw new Error(`Error fetching image: ${error}`); + console.error(error); + // TODO: Handle `Input buffer contains unsupported image format` error from sharp when the image + // is actually a video or another media file. + let exitCode = 1; + if ( + error.cause && + (error.cause.code == 'UND_ERR_HEADERS_TIMEOUT' || + error.cause.code == 'UND_ERR_BODY_TIMEOUT' || + error.cause.code == 'UND_ERR_CONNECT_TIMEOUT' || + error.cause.code == 'ECONNRESET') + ) { + exitCode = 2; + } else if ( + error.cause && + error.cause.code == 'UND_ERR_RESPONSE_STATUS_CODE' && + error.cause.statusCode === 429 + ) { + exitCode = 3; + } + process.exit(exitCode); }); diff --git a/src/token-processor/queue/job/process-token-job.ts b/src/token-processor/queue/job/process-token-job.ts index 4e190e13..cdaba8dd 100644 --- a/src/token-processor/queue/job/process-token-job.ts +++ b/src/token-processor/queue/job/process-token-job.ts @@ -16,7 +16,7 @@ import { StacksNodeRpcClient } from '../../stacks-node/stacks-node-rpc-client'; import { StacksNodeClarityError, TooManyRequestsHttpError } from '../../util/errors'; import { fetchAllMetadataLocalesFromBaseUri, - getFetchableUrl, + getFetchableDecentralizedStorageUrl, getTokenSpecificUri, } from '../../util/metadata-helpers'; import { RetryableJobError } from '../errors'; @@ -214,7 +214,7 @@ export class ProcessTokenJob extends Job { return; } // Before we return the uri, check if its fetchable hostname is not already rate limited. - const fetchable = getFetchableUrl(uri); + const fetchable = getFetchableDecentralizedStorageUrl(uri); const rateLimitedHost = await this.db.getRateLimitedHost({ hostname: fetchable.hostname }); if (rateLimitedHost) { const retryAfter = Date.parse(rateLimitedHost.retry_after); diff --git a/src/token-processor/util/image-cache.ts b/src/token-processor/util/image-cache.ts index 9ad70e42..1c11aac9 100644 --- a/src/token-processor/util/image-cache.ts +++ b/src/token-processor/util/image-cache.ts @@ -1,33 +1,81 @@ import * as child_process from 'child_process'; import { ENV } from '../../env'; -import { MetadataParseError } from './errors'; -import { parseDataUrl, getFetchableUrl } from './metadata-helpers'; +import { MetadataParseError, MetadataTimeoutError, TooManyRequestsHttpError } from './errors'; +import { parseDataUrl, getFetchableDecentralizedStorageUrl } from './metadata-helpers'; import { logger } from '@hirosystems/api-toolkit'; import { PgStore } from '../../pg/pg-store'; +import { errors } from 'undici'; +import { RetryableJobError } from '../queue/errors'; /** - * If an external image processor script is configured, then it will process the given image URL for - * the purpose of caching on a CDN (or whatever else it may be created to do). The script is - * expected to return a new URL for the image. If the script is not configured, then the original - * URL is returned immediately. If a data-uri is passed, it is also immediately returned without - * being passed to the script. + * If an external image processor script is configured in the `METADATA_IMAGE_CACHE_PROCESSOR` ENV + * var, this function will process the given image URL for the purpose of caching on a CDN (or + * whatever else it may be created to do). The script is expected to return a new URL for the image + * via `stdout`, with an optional 2nd line with another URL for a thumbnail version of the same + * cached image. If the script is not configured, then the original URL is returned immediately. If + * a data-uri is passed, it is also immediately returned without being passed to the script. + * + * The Image Cache script must return a status code of `0` to mark a successful cache. Other code + * returns available are: + * * `1`: A generic error occurred. Cache should not be retried. + * * `2`: Image fetch timed out before caching was possible. Should be retried. + * * `3`: Image fetch failed due to rate limits from the remote server. Should be retried. */ -export async function processImageUrl( +export async function processImageCache( imgUrl: string, contractPrincipal: string, tokenNumber: bigint ): Promise { const imageCacheProcessor = ENV.METADATA_IMAGE_CACHE_PROCESSOR; - if (!imageCacheProcessor) { - return [imgUrl]; - } - if (imgUrl.startsWith('data:')) { - return [imgUrl]; + if (!imageCacheProcessor || imgUrl.startsWith('data:')) return [imgUrl]; + logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`); + const { code, stdout, stderr } = await callImageCacheScript( + imageCacheProcessor, + imgUrl, + contractPrincipal, + tokenNumber + ); + switch (code) { + case 0: + try { + const urls = stdout + .trim() + .split('\n') + .map(r => new URL(r).toString()); + logger.info(urls, `ImageCache processed token ${contractPrincipal} (${tokenNumber})`); + return urls; + } catch (error) { + // The script returned a code `0` but the results are invalid. This could happen because of + // an unknown script error so we should mark it as retryable. + throw new RetryableJobError( + `ImageCache unknown error`, + new Error(`Invalid cached url for ${imgUrl}: ${stdout}, stderr: ${stderr}`) + ); + } + case 2: + throw new RetryableJobError(`ImageCache fetch timed out`, new MetadataTimeoutError(imgUrl)); + case 3: + throw new RetryableJobError( + `ImageCache fetch rate limited`, + new TooManyRequestsHttpError(new URL(imgUrl), new errors.ResponseStatusCodeError()) + ); + default: + throw new Error(`ImageCache script error (code ${code}): ${stderr}`); } +} - logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`); +async function callImageCacheScript( + imageCacheProcessor: string, + imgUrl: string, + contractPrincipal: string, + tokenNumber: bigint +): Promise<{ + code: number; + stdout: string; + stderr: string; +}> { const repoDir = process.cwd(); - const { code, stdout, stderr } = await new Promise<{ + return await new Promise<{ code: number; stdout: string; stderr: string; @@ -37,26 +85,24 @@ export async function processImageUrl( [imgUrl, contractPrincipal, tokenNumber.toString()], { cwd: repoDir } ); + let code = 0; let stdout = ''; let stderr = ''; cp.stdout.on('data', data => (stdout += data)); cp.stderr.on('data', data => (stderr += data)); - cp.on('close', code => resolve({ code: code ?? 0, stdout, stderr })); + cp.on('close', _ => resolve({ code, stdout, stderr })); + cp.on('exit', processCode => { + code = processCode ?? 0; + }); }); - if (code !== 0) throw new Error(`ImageCache error: ${stderr}`); - const result = stdout.trim().split('\n'); - logger.info(result, `ImageCache processed token ${contractPrincipal} (${tokenNumber})`); - - try { - return result.map(r => new URL(r).toString()); - } catch (error) { - throw new Error( - `Image processing script returned an invalid url for ${imgUrl}: ${result}, stderr: ${stderr}` - ); - } } -export function getImageUrl(uri: string): string { +/** + * Converts a raw image URI from metadata into a fetchable URL. + * @param uri - Original image URI + * @returns Normalized URL string + */ +export function normalizeImageUri(uri: string): string { // Support images embedded in a Data URL if (uri.startsWith('data:')) { const dataUrl = parseDataUrl(uri); @@ -68,7 +114,7 @@ export function getImageUrl(uri: string): string { } return uri; } - const fetchableUrl = getFetchableUrl(uri); + const fetchableUrl = getFetchableDecentralizedStorageUrl(uri); return fetchableUrl.toString(); } @@ -81,8 +127,8 @@ export async function reprocessTokenImageCache( const imageUris = await db.getTokenImageUris(contractPrincipal, tokenIds); for (const token of imageUris) { try { - const [cached, thumbnail] = await processImageUrl( - getFetchableUrl(token.image).toString(), + const [cached, thumbnail] = await processImageCache( + getFetchableDecentralizedStorageUrl(token.image).toString(), contractPrincipal, BigInt(token.token_number) ); diff --git a/src/token-processor/util/metadata-helpers.ts b/src/token-processor/util/metadata-helpers.ts index c53548e8..8bfbd37f 100644 --- a/src/token-processor/util/metadata-helpers.ts +++ b/src/token-processor/util/metadata-helpers.ts @@ -18,7 +18,7 @@ import { TooManyRequestsHttpError, } from './errors'; import { RetryableJobError } from '../queue/errors'; -import { getImageUrl, processImageUrl } from './image-cache'; +import { normalizeImageUri, processImageCache } from './image-cache'; import { RawMetadataLocale, RawMetadataLocalizationCType, @@ -134,8 +134,8 @@ async function parseMetadataForInsertion( let cachedImage: string | undefined; let cachedThumbnailImage: string | undefined; if (image && typeof image === 'string') { - const normalizedUrl = getImageUrl(image); - [cachedImage, cachedThumbnailImage] = await processImageUrl( + const normalizedUrl = normalizeImageUri(image); + [cachedImage, cachedThumbnailImage] = await processImageCache( normalizedUrl, contract.principal, token.token_number @@ -255,7 +255,7 @@ export async function getMetadataFromUri(token_uri: string): Promise { const contract = 'SP3QSAJQ4EA8WXEDSRRKMZZ29NH91VZ6C5X88FGZQ.crashpunks-v2'; @@ -11,7 +11,7 @@ describe('Image cache', () => { }); test('transforms image URL correctly', async () => { - const transformed = await processImageUrl(url, contract, tokenNumber); + const transformed = await processImageCache(url, contract, tokenNumber); expect(transformed).toStrictEqual([ 'http://cloudflare-ipfs.com/test/image.png?processed=true', 'http://cloudflare-ipfs.com/test/image.png?processed=true&thumb=true', @@ -20,18 +20,20 @@ describe('Image cache', () => { test('ignores data: URL', async () => { const url = 'data:123456'; - const transformed = await processImageUrl(url, contract, tokenNumber); + const transformed = await processImageCache(url, contract, tokenNumber); expect(transformed).toStrictEqual(['data:123456']); }); test('ignores empty script paths', async () => { ENV.METADATA_IMAGE_CACHE_PROCESSOR = ''; - const transformed = await processImageUrl(url, contract, tokenNumber); + const transformed = await processImageCache(url, contract, tokenNumber); expect(transformed).toStrictEqual([url]); }); test('handles script errors', async () => { ENV.METADATA_IMAGE_CACHE_PROCESSOR = './tests/test-image-cache-error.js'; - await expect(processImageUrl(url, contract, tokenNumber)).rejects.toThrow(/ImageCache error/); + await expect(processImageCache(url, contract, tokenNumber)).rejects.toThrow( + /ImageCache script error/ + ); }); }); diff --git a/tests/metadata-helpers.test.ts b/tests/metadata-helpers.test.ts index 5edb9912..dc6ec47d 100644 --- a/tests/metadata-helpers.test.ts +++ b/tests/metadata-helpers.test.ts @@ -7,7 +7,7 @@ import { MetadataTimeoutError, } from '../src/token-processor/util/errors'; import { - getFetchableUrl, + getFetchableDecentralizedStorageUrl, getMetadataFromUri, getTokenSpecificUri, fetchMetadata, @@ -187,16 +187,16 @@ describe('Metadata Helpers', () => { ENV.PUBLIC_GATEWAY_IPFS = 'https://cloudflare-ipfs.com'; ENV.PUBLIC_GATEWAY_ARWEAVE = 'https://arweave.net'; const arweave = 'ar://II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json'; - expect(getFetchableUrl(arweave).toString()).toBe( + expect(getFetchableDecentralizedStorageUrl(arweave).toString()).toBe( 'https://arweave.net/II4z2ziYyqG7-kWDa98lWGfjxRdYOx9Zdld9P_I_kzE/9731.json' ); const ipfs = 'ipfs://ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json'; - expect(getFetchableUrl(ipfs).toString()).toBe( + expect(getFetchableDecentralizedStorageUrl(ipfs).toString()).toBe( 'https://cloudflare-ipfs.com/ipfs/bafybeifwoqwdhs5djtx6vopvuwfcdrqeuecayp5wzpzjylxycejnhtrhgu/vague_art_paintings/vague_art_paintings_6_metadata.json' ); const ipfs2 = 'ipfs://QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png'; - expect(getFetchableUrl(ipfs2).toString()).toBe( + expect(getFetchableDecentralizedStorageUrl(ipfs2).toString()).toBe( 'https://cloudflare-ipfs.com/ipfs/QmYCnfeseno5cLpC75rmy6LQhsNYQCJabiuwqNUXMaA3Fo/1145.png' ); }); diff --git a/tests/test-image-cache-error.js b/tests/test-image-cache-error.js index 4ab0feb6..878b3073 100755 --- a/tests/test-image-cache-error.js +++ b/tests/test-image-cache-error.js @@ -1,3 +1,3 @@ #!/usr/bin/env node console.error('Test error'); -throw new Error('Test'); +process.exit(1); From b78560df7f2c6228ed68ad8b8949ffcafc81cc7c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 13 May 2024 16:35:54 +0000 Subject: [PATCH 10/11] chore(release): 0.7.0-beta.5 [skip ci] ## [0.7.0-beta.5](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.4...v0.7.0-beta.5) (2024-05-13) ### Bug Fixes * improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59ac9ecb..552e4144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [0.7.0-beta.5](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.4...v0.7.0-beta.5) (2024-05-13) + + +### Bug Fixes + +* improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62)) + ## [0.7.0-beta.4](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.3...v0.7.0-beta.4) (2024-05-08) From 1aa1603771acd782ea1c9094f4b34fe7dc92455c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 13 May 2024 16:53:11 +0000 Subject: [PATCH 11/11] chore(release): 0.7.0 [skip ci] ## [0.7.0](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0) (2024-05-13) ### Features * add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a)) * update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f)) * upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51)) ### Bug Fixes * get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422)) * get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb)) * image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43)) * improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62)) * reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738)) --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 552e4144..06bc81cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## [0.7.0](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0) (2024-05-13) + + +### Features + +* add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a)) +* update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f)) +* upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51)) + + +### Bug Fixes + +* get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422)) +* get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb)) +* image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43)) +* improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62)) +* reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738)) + ## [0.7.0-beta.5](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.4...v0.7.0-beta.5) (2024-05-13)