Skip to content

Commit

Permalink
Merge pull request #227 from hirosystems/master
Browse files Browse the repository at this point in the history 
merge master into develop
  • Loading branch information
@rafaelcr
rafaelcr authored Jul 16, 2024
2 parents 0196466 + 1aa1603 commit 0fafe1c
Show file tree
Hide file tree
Showing 8 changed files with 235 additions and 86 deletions.
60 changes: 60 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,63 @@
## [0.7.0](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0) (2024-05-13)


### Features

* add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a))
* update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f))
* upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51))


### Bug Fixes

* get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422))
* get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb))
* image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43))
* improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62))
* reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738))

## [0.7.0-beta.5](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.4...v0.7.0-beta.5) (2024-05-13)


### Bug Fixes

* improve image cache error handling ([#214](https://github.com/hirosystems/token-metadata-api/issues/214)) ([115a745](https://github.com/hirosystems/token-metadata-api/commit/115a745c268e7bb8115a488ca111e8b46cefed62))

## [0.7.0-beta.4](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.3...v0.7.0-beta.4) (2024-05-08)


### Bug Fixes

* get access token properly ([a6b98c5](https://github.com/hirosystems/token-metadata-api/commit/a6b98c5099a9de1d88e74eed66dece1c4c157422))

## [0.7.0-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.2...v0.7.0-beta.3) (2024-05-07)


### Bug Fixes

* image cache agent arg types ([5826628](https://github.com/hirosystems/token-metadata-api/commit/5826628a329225fbf697a092dc201fc74fb96d43))

## [0.7.0-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v0.7.0-beta.1...v0.7.0-beta.2) (2024-05-07)


### Bug Fixes

* reuse gcs token and validate image cache script errors ([#213](https://github.com/hirosystems/token-metadata-api/issues/213)) ([5e1af5c](https://github.com/hirosystems/token-metadata-api/commit/5e1af5c28cd0b1313f78a59b015669ceb07e5738))

## [0.7.0-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v0.6.3...v0.7.0-beta.1) (2024-05-07)


### Features

* add admin rpc to reprocess token image cache ([#205](https://github.com/hirosystems/token-metadata-api/issues/205)) ([2fdcb33](https://github.com/hirosystems/token-metadata-api/commit/2fdcb33908062770da4e334810fd04bb378db66a))
* update ts client with image thumbnails ([#206](https://github.com/hirosystems/token-metadata-api/issues/206)) ([c24cb56](https://github.com/hirosystems/token-metadata-api/commit/c24cb56b854123b252eb2e2616bb8589c5b36f0f))
* upload token images to gcs ([#204](https://github.com/hirosystems/token-metadata-api/issues/204)) ([1cec219](https://github.com/hirosystems/token-metadata-api/commit/1cec2195a2b3df9e9c85f0152732594caa8c8c51))


### Bug Fixes

* get gcs auth token dynamically for image cache ([#210](https://github.com/hirosystems/token-metadata-api/issues/210)) ([8434b22](https://github.com/hirosystems/token-metadata-api/commit/8434b229f6d38e6799bf84bd6f1eb4de106996bb))

## [0.6.3](https://github.com/hirosystems/token-metadata-api/compare/v0.6.2...v0.6.3) (2024-05-07)


Expand Down
102 changes: 68 additions & 34 deletions config/image-cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ const IMAGE_RESIZE_WIDTH = parseInt(process.env['IMAGE_CACHE_RESIZE_WIDTH'] ?? '
const GCS_BUCKET_NAME = process.env['IMAGE_CACHE_GCS_BUCKET_NAME'];
const GCS_OBJECT_NAME_PREFIX = process.env['IMAGE_CACHE_GCS_OBJECT_NAME_PREFIX'];
const CDN_BASE_PATH = process.env['IMAGE_CACHE_CDN_BASE_PATH'];
const TIMEOUT = parseInt(process.env['METADATA_FETCH_TIMEOUT_MS'] ?? '30000');
const MAX_REDIRECTIONS = parseInt(process.env['METADATA_FETCH_MAX_REDIRECTIONS'] ?? '0');
const MAX_RESPONSE_SIZE = parseInt(process.env['IMAGE_CACHE_MAX_BYTE_SIZE'] ?? '-1');

async function getGcsAuthToken() {
const envToken = process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'];
Expand All @@ -44,49 +47,46 @@ async function getGcsAuthToken() {
{
method: 'GET',
headers: { 'Metadata-Flavor': 'Google' },
throwOnError: true,
}
);
if (response.data?.access_token) return response.data.access_token;
throw new Error(`GCS token not found`);
const json = await response.body.json();
// Cache the token so we can reuse it for other images.
process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = json.access_token;
return json.access_token;
} catch (error) {
throw new Error(`Error fetching GCS access token: ${error.message}`);
throw new Error(`GCS access token error: ${error}`);
}
}

async function upload(stream, name, authToken) {
try {
const response = await request(
`https://storage.googleapis.com/upload/storage/v1/b/${GCS_BUCKET_NAME}/o?uploadType=media&name=${GCS_OBJECT_NAME_PREFIX}${name}`,
{
method: 'POST',
body: stream,
headers: { 'Content-Type': 'image/png', Authorization: `Bearer ${authToken}` },
}
);
if (response.statusCode !== 200) throw new Error(`GCS error: ${response.statusCode}`);
return `${CDN_BASE_PATH}${name}`;
} catch (error) {
throw new Error(`Error uploading ${name}: ${error.message}`);
}
await request(
`https://storage.googleapis.com/upload/storage/v1/b/${GCS_BUCKET_NAME}/o?uploadType=media&name=${GCS_OBJECT_NAME_PREFIX}${name}`,
{
method: 'POST',
body: stream,
headers: { 'Content-Type': 'image/png', Authorization: `Bearer ${authToken}` },
throwOnError: true,
}
);
return `${CDN_BASE_PATH}${name}`;
}

fetch(
IMAGE_URL,
{
dispatcher: new Agent({
headersTimeout: process.env['METADATA_FETCH_TIMEOUT_MS'],
bodyTimeout: process.env['METADATA_FETCH_TIMEOUT_MS'],
maxRedirections: process.env['METADATA_FETCH_MAX_REDIRECTIONS'],
maxResponseSize: process.env['IMAGE_CACHE_MAX_BYTE_SIZE'],
headersTimeout: TIMEOUT,
bodyTimeout: TIMEOUT,
maxRedirections: MAX_REDIRECTIONS,
maxResponseSize: MAX_RESPONSE_SIZE,
throwOnError: true,
connect: {
rejectUnauthorized: false, // Ignore SSL cert errors.
},
}),
},
({ statusCode, body }) => {
if (statusCode !== 200) throw new Error(`Failed to fetch image: ${statusCode}`);
return body;
}
({ body }) => body
)
.then(async response => {
const imageReadStream = Readable.fromWeb(response.body);
Expand All @@ -99,15 +99,49 @@ fetch(
passThrough.pipe(fullSizeTransform);
passThrough.pipe(thumbnailTransform);

const authToken = await getGcsAuthToken();
const results = await Promise.all([
upload(fullSizeTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}.png`, authToken),
upload(thumbnailTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}-thumb.png`, authToken),
]);

// The API will read these strings as CDN URLs.
for (const result of results) console.log(result);
let didRetryUnauthorized = false;
while (true) {
const authToken = await getGcsAuthToken();
try {
const results = await Promise.all([
upload(fullSizeTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}.png`, authToken),
upload(thumbnailTransform, `${CONTRACT_PRINCIPAL}/${TOKEN_NUMBER}-thumb.png`, authToken),
]);
for (const r of results) console.log(r);
break;
} catch (error) {
if (
!didRetryUnauthorized &&
error.cause &&
error.cause.code == 'UND_ERR_RESPONSE_STATUS_CODE' &&
(error.cause.statusCode === 401 || error.cause.statusCode === 403)
) {
// GCS token is probably expired. Force a token refresh before trying again.
process.env['IMAGE_CACHE_GCS_AUTH_TOKEN'] = undefined;
didRetryUnauthorized = true;
} else throw error;
}
}
})
.catch(error => {
throw new Error(`Error processing image: ${error.message}`);
console.error(error);
// TODO: Handle `Input buffer contains unsupported image format` error from sharp when the image
// is actually a video or another media file.
let exitCode = 1;
if (
error.cause &&
(error.cause.code == 'UND_ERR_HEADERS_TIMEOUT' ||
error.cause.code == 'UND_ERR_BODY_TIMEOUT' ||
error.cause.code == 'UND_ERR_CONNECT_TIMEOUT' ||
error.cause.code == 'ECONNRESET')
) {
exitCode = 2;
} else if (
error.cause &&
error.cause.code == 'UND_ERR_RESPONSE_STATUS_CODE' &&
error.cause.statusCode === 429
) {
exitCode = 3;
}
process.exit(exitCode);
});
4 changes: 2 additions & 2 deletions src/token-processor/queue/job/process-token-job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { StacksNodeRpcClient } from '../../stacks-node/stacks-node-rpc-client';
import { StacksNodeClarityError, TooManyRequestsHttpError } from '../../util/errors';
import {
fetchAllMetadataLocalesFromBaseUri,
getFetchableUrl,
getFetchableDecentralizedStorageUrl,
getTokenSpecificUri,
} from '../../util/metadata-helpers';
import { RetryableJobError } from '../errors';
Expand Down Expand Up @@ -214,7 +214,7 @@ export class ProcessTokenJob extends Job {
return;
}
// Before we return the uri, check if its fetchable hostname is not already rate limited.
const fetchable = getFetchableUrl(uri);
const fetchable = getFetchableDecentralizedStorageUrl(uri);
const rateLimitedHost = await this.db.getRateLimitedHost({ hostname: fetchable.hostname });
if (rateLimitedHost) {
const retryAfter = Date.parse(rateLimitedHost.retry_after);
Expand Down
112 changes: 79 additions & 33 deletions src/token-processor/util/image-cache.ts
View file
Original file line number Diff line number Diff line change
@@ -1,62 +1,108 @@
import * as child_process from 'child_process';
import { ENV } from '../../env';
import { MetadataParseError } from './errors';
import { parseDataUrl, getFetchableUrl } from './metadata-helpers';
import { MetadataParseError, MetadataTimeoutError, TooManyRequestsHttpError } from './errors';
import { parseDataUrl, getFetchableDecentralizedStorageUrl } from './metadata-helpers';
import { logger } from '@hirosystems/api-toolkit';
import { PgStore } from '../../pg/pg-store';
import { errors } from 'undici';
import { RetryableJobError } from '../queue/errors';

/**
* If an external image processor script is configured, then it will process the given image URL for
* the purpose of caching on a CDN (or whatever else it may be created to do). The script is
* expected to return a new URL for the image. If the script is not configured, then the original
* URL is returned immediately. If a data-uri is passed, it is also immediately returned without
* being passed to the script.
* If an external image processor script is configured in the `METADATA_IMAGE_CACHE_PROCESSOR` ENV
* var, this function will process the given image URL for the purpose of caching on a CDN (or
* whatever else it may be created to do). The script is expected to return a new URL for the image
* via `stdout`, with an optional 2nd line with another URL for a thumbnail version of the same
* cached image. If the script is not configured, then the original URL is returned immediately. If
* a data-uri is passed, it is also immediately returned without being passed to the script.
*
* The Image Cache script must return a status code of `0` to mark a successful cache. Other code
* returns available are:
* * `1`: A generic error occurred. Cache should not be retried.
* * `2`: Image fetch timed out before caching was possible. Should be retried.
* * `3`: Image fetch failed due to rate limits from the remote server. Should be retried.
*/
export async function processImageUrl(
export async function processImageCache(
imgUrl: string,
contractPrincipal: string,
tokenNumber: bigint
): Promise<string[]> {
const imageCacheProcessor = ENV.METADATA_IMAGE_CACHE_PROCESSOR;
if (!imageCacheProcessor) {
return [imgUrl];
}
if (imgUrl.startsWith('data:')) {
return [imgUrl];
if (!imageCacheProcessor || imgUrl.startsWith('data:')) return [imgUrl];
logger.info(`ImageCache processing token ${contractPrincipal} (${tokenNumber}) at ${imgUrl}`);
const { code, stdout, stderr } = await callImageCacheScript(
imageCacheProcessor,
imgUrl,
contractPrincipal,
tokenNumber
);
switch (code) {
case 0:
try {
const urls = stdout
.trim()
.split('\n')
.map(r => new URL(r).toString());
logger.info(urls, `ImageCache processed token ${contractPrincipal} (${tokenNumber})`);
return urls;
} catch (error) {
// The script returned a code `0` but the results are invalid. This could happen because of
// an unknown script error so we should mark it as retryable.
throw new RetryableJobError(
`ImageCache unknown error`,
new Error(`Invalid cached url for ${imgUrl}: ${stdout}, stderr: ${stderr}`)
);
}
case 2:
throw new RetryableJobError(`ImageCache fetch timed out`, new MetadataTimeoutError(imgUrl));
case 3:
throw new RetryableJobError(
`ImageCache fetch rate limited`,
new TooManyRequestsHttpError(new URL(imgUrl), new errors.ResponseStatusCodeError())
);
default:
throw new Error(`ImageCache script error (code ${code}): ${stderr}`);
}
logger.info(`ImageCache processing image for token ${contractPrincipal} (${tokenNumber})...`);
}

async function callImageCacheScript(
imageCacheProcessor: string,
imgUrl: string,
contractPrincipal: string,
tokenNumber: bigint
): Promise<{
code: number;
stdout: string;
stderr: string;
}> {
const repoDir = process.cwd();
const { code, stdout, stderr } = await new Promise<{
return await new Promise<{
code: number;
stdout: string;
stderr: string;
}>((resolve, reject) => {
}>(resolve => {
const cp = child_process.spawn(
imageCacheProcessor,
[imgUrl, contractPrincipal, tokenNumber.toString()],
{ cwd: repoDir }
);
let code = 0;
let stdout = '';
let stderr = '';
cp.stdout.on('data', data => (stdout += data));
cp.stderr.on('data', data => (stderr += data));
cp.on('close', code => resolve({ code: code ?? 0, stdout, stderr }));
cp.on('error', error => reject(error));
cp.on('close', _ => resolve({ code, stdout, stderr }));
cp.on('exit', processCode => {
code = processCode ?? 0;
});
});
if (code !== 0 && stderr) {
logger.warn(stderr, `ImageCache error`);
}
const result = stdout.trim().split('\n');
try {
return result.map(r => new URL(r).toString());
} catch (error) {
throw new Error(
`Image processing script returned an invalid url for ${imgUrl}: ${result}, stderr: ${stderr}`
);
}
}

export function getImageUrl(uri: string): string {
/**
* Converts a raw image URI from metadata into a fetchable URL.
* @param uri - Original image URI
* @returns Normalized URL string
*/
export function normalizeImageUri(uri: string): string {
// Support images embedded in a Data URL
if (uri.startsWith('data:')) {
const dataUrl = parseDataUrl(uri);
Expand All @@ -68,7 +114,7 @@ export function getImageUrl(uri: string): string {
}
return uri;
}
const fetchableUrl = getFetchableUrl(uri);
const fetchableUrl = getFetchableDecentralizedStorageUrl(uri);
return fetchableUrl.toString();
}

Expand All @@ -81,8 +127,8 @@ export async function reprocessTokenImageCache(
const imageUris = await db.getTokenImageUris(contractPrincipal, tokenIds);
for (const token of imageUris) {
try {
const [cached, thumbnail] = await processImageUrl(
getFetchableUrl(token.image).toString(),
const [cached, thumbnail] = await processImageCache(
getFetchableDecentralizedStorageUrl(token.image).toString(),
contractPrincipal,
BigInt(token.token_number)
);
Expand Down
Loading

0 comments on commit 0fafe1c

Please sign in to comment.