Skip to content

Commit

Permalink
feat(blob): Provide onUploadProgress({ loaded, total, percentage }) (#…
Browse files Browse the repository at this point in the history 
…782)

* feat(blob): Provide onUploadProgress({ loaded, total, percentage })

This commit introduces an `onUploadProgress` callback to put/upload*.

Here's how to use it:
```ts
// also works with upload()
const blob = await put('file.pdf', file, {
  onUploadProgress(event) {
    console.log(event.loaded, event.total, event.percentage);
  }
});
```
  • Loading branch information
@vvo
vvo authored Nov 6, 2024
1 parent 6f654d9 commit c3afec3
Show file tree
Hide file tree
Showing 34 changed files with 2,432 additions and 895 deletions.
25 changes: 25 additions & 0 deletions .changeset/brown-years-heal.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
'@vercel/blob': minor
'vercel-storage-integration-test-suite': minor
---

Add onUploadProgress feature to put/upload

You can now track the upload progress in Node.js and all major browsers when
using put/upload in multipart, non-multipart and client upload modes. Basically
anywhere in our API you can upload a file, then you can follow the upload
progress.

Here's a basic usage example:

```
const blob = await put('big-file.pdf', file, {
access: 'public',
onUploadProgress(event) {
console.log(event.loaded, event.total, event.percentage);
}
});
```

Fixes #543
Fixes #642
8 changes: 7 additions & 1 deletion packages/blob/jest/setup.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,11 @@
// but they are available everywhere else.
// See https://stackoverflow.com/questions/68468203/why-am-i-getting-textencoder-is-not-defined-in-jest
const { TextEncoder, TextDecoder } = require('node:util');
// eslint-disable-next-line import/order -- On purpose to make requiring undici work
const { ReadableStream } = require('node:stream/web');

Object.assign(global, { TextDecoder, TextEncoder });
Object.assign(global, { TextDecoder, TextEncoder, ReadableStream });

const { Request, Response } = require('undici');

Object.assign(global, { Request, Response });
10 changes: 6 additions & 4 deletions packages/blob/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,22 +63,24 @@
"async-retry": "^1.3.3",
"bytes": "^3.1.2",
"is-buffer": "^2.0.5",
"is-node-process": "^1.2.0",
"throttleit": "^2.1.0",
"undici": "^5.28.4"
},
"devDependencies": {
"@edge-runtime/jest-environment": "2.3.10",
"@edge-runtime/types": "2.2.9",
"@types/async-retry": "1.4.8",
"@types/async-retry": "1.4.9",
"@types/bytes": "3.1.4",
"@types/jest": "29.5.13",
"@types/node": "22.7.3",
"@types/jest": "29.5.14",
"@types/node": "22.9.0",
"eslint": "8.56.0",
"eslint-config-custom": "workspace:*",
"jest": "29.7.0",
"jest-environment-jsdom": "29.7.0",
"ts-jest": "29.2.5",
"tsconfig": "workspace:*",
"tsup": "8.3.0"
"tsup": "8.3.5"
},
"engines": {
"node": ">=16.14"
Expand Down
162 changes: 125 additions & 37 deletions packages/blob/src/api.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,20 @@
import type { RequestInit, Response } from 'undici';
import { fetch } from 'undici';
import type { Response } from 'undici';
import retry from 'async-retry';
import isNetworkError from './is-network-error';
import { debug } from './debug';
import type { BlobCommandOptions } from './helpers';
import { BlobError, getTokenFromOptionsOrEnv } from './helpers';
import type {
BlobCommandOptions,
BlobRequestInit,
WithUploadProgress,
} from './helpers';
import {
BlobError,
computeBodyLength,
getApiUrl,
getTokenFromOptionsOrEnv,
} from './helpers';
import { blobRequest } from './request';
import { DOMException } from './dom-exception';

// maximum pathname length is:
// 1024 (provider limit) - 26 chars (vercel internal suffixes) - 31 chars (blob `-randomId` suffix) = 967
Expand Down Expand Up @@ -132,20 +143,6 @@ function getApiVersion(): string {
return `${versionOverride ?? BLOB_API_VERSION}`;
}

function getApiUrl(pathname = ''): string {
let baseUrl = null;
try {
// wrapping this code in a try/catch as this function is used in the browser and Vite doesn't define the process.env.
// As this varaible is NOT used in production, it will always default to production endpoint
baseUrl =
process.env.VERCEL_BLOB_API_URL ||
process.env.NEXT_PUBLIC_VERCEL_BLOB_API_URL;
} catch {
// noop
}
return `${baseUrl || 'https://blob.vercel-storage.com'}${pathname}`;
}

function getRetries(): number {
try {
const retries = process.env.VERCEL_BLOB_RETRIES || '10';
Expand Down Expand Up @@ -175,7 +172,6 @@ async function getBlobError(

try {
const data = (await response.json()) as BlobApiError;

code = data.error?.code ?? 'unknown_error';
message = data.error?.message;
} catch {
Expand Down Expand Up @@ -254,8 +250,8 @@ async function getBlobError(

export async function requestApi<TResponse>(
pathname: string,
init: RequestInit,
commandOptions: BlobCommandOptions | undefined,
init: BlobRequestInit,
commandOptions: (BlobCommandOptions & WithUploadProgress) | undefined,
): Promise<TResponse> {
const apiVersion = getApiVersion();
const token = getTokenFromOptionsOrEnv(commandOptions);
Expand All @@ -264,23 +260,75 @@ export async function requestApi<TResponse>(
const [, , , storeId = ''] = token.split('_');
const requestId = `${storeId}:${Date.now()}:${Math.random().toString(16).slice(2)}`;
let retryCount = 0;
let bodyLength = 0;
let totalLoaded = 0;
const sendBodyLength =
commandOptions?.onUploadProgress || shouldUseXContentLength();

if (
init.body &&
// 1. For upload progress we always need to know the total size of the body
// 2. In development we need the header for put() to work correctly when passing a stream
sendBodyLength
) {
bodyLength = computeBodyLength(init.body);
}

if (commandOptions?.onUploadProgress) {
commandOptions.onUploadProgress({
loaded: 0,
total: bodyLength,
percentage: 0,
});
}

const apiResponse = await retry(
async (bail) => {
let res: Response;

// try/catch here to treat certain errors as not-retryable
try {
res = await fetch(getApiUrl(pathname), {
...init,
headers: {
'x-api-blob-request-id': requestId,
'x-api-blob-request-attempt': String(retryCount),
'x-api-version': apiVersion,
authorization: `Bearer ${token}`,
...extraHeaders,
...init.headers,
res = await blobRequest({
input: getApiUrl(pathname),
init: {
...init,
headers: {
'x-api-blob-request-id': requestId,
'x-api-blob-request-attempt': String(retryCount),
'x-api-version': apiVersion,
...(sendBodyLength
? { 'x-content-length': String(bodyLength) }
: {}),
authorization: `Bearer ${token}`,
...extraHeaders,
...init.headers,
},
},
onUploadProgress: commandOptions?.onUploadProgress
? (loaded) => {
const total = bodyLength !== 0 ? bodyLength : loaded;
totalLoaded = loaded;
const percentage =
bodyLength > 0
? Number(((loaded / total) * 100).toFixed(2))
: 0;

// Leave percentage 100 for the end of request
if (percentage === 100 && bodyLength > 0) {
return;
}

commandOptions.onUploadProgress?.({
loaded,
// When passing a stream to put(), we have no way to know the total size of the body.
// Instead of defining total as total?: number we decided to set the total to the currently
// loaded number. This is not inaccurate and way more practical for DX.
// Passing down a stream to put() is very rare
total,
percentage,
});
}
: undefined,
});
} catch (error) {
// if the request was aborted, don't retry
Expand All @@ -289,6 +337,18 @@ export async function requestApi<TResponse>(
return;
}

// We specifically target network errors because fetch network errors are regular TypeErrors
// We want to retry for network errors, but not for other TypeErrors
if (isNetworkError(error)) {
throw error;
}

// If we messed up the request part, don't even retry
if (error instanceof TypeError) {
bail(error);
return;
}

// retry for any other erros thrown by fetch
throw error;
}
Expand All @@ -314,7 +374,10 @@ export async function requestApi<TResponse>(
{
retries: getRetries(),
onRetry: (error) => {
debug(`retrying API request to ${pathname}`, error.message);
if (error instanceof Error) {
debug(`retrying API request to ${pathname}`, error.message);
}

retryCount = retryCount + 1;
},
},
Expand All @@ -324,6 +387,20 @@ export async function requestApi<TResponse>(
throw new BlobUnknownError();
}

// Calling onUploadProgress here has two benefits:
// 1. It ensures 100% is only reached at the end of the request. While otherwise you can reach 100%
// before the request is fully done, as we only really measure what gets sent over the wire, not what
// has been processed by the server.
// 2. It makes the uploadProgress "work" even in rare cases where fetch/xhr onprogress is not working
// And in the case of multipart uploads it actually provides a simple progress indication (per part)
if (commandOptions?.onUploadProgress) {
commandOptions.onUploadProgress({
loaded: totalLoaded,
total: totalLoaded,
percentage: 100,
});
}

return (await apiResponse.json()) as TResponse;
}

Expand All @@ -333,20 +410,31 @@ function getProxyThroughAlternativeApiHeaderFromEnv(): {
const extraHeaders: Record<string, string> = {};

try {
if ('VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env) {
if (
'VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env &&
process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API !== undefined
) {
extraHeaders['x-proxy-through-alternative-api'] =
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- we know it's here from the if
process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API!;
process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API;
} else if (
'NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env
'NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env &&
process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API !==
undefined
) {
extraHeaders['x-proxy-through-alternative-api'] =
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- we know it's here from the if
process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API!;
process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API;
}
} catch {
// noop
}

return extraHeaders;
}

function shouldUseXContentLength(): boolean {
try {
return process.env.VERCEL_BLOB_USE_X_CONTENT_LENGTH === '1';
} catch {
return false;
}
}
5 changes: 0 additions & 5 deletions packages/blob/src/client.browser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ describe('client', () => {
'https://blob.vercel-storage.com/foo.txt',
{
body: 'Test file data',
duplex: 'half',
headers: {
authorization: 'Bearer vercel_blob_client_fake_123',
'x-api-blob-request-attempt': '0',
Expand Down Expand Up @@ -232,7 +231,6 @@ describe('client', () => {
'x-mpu-part-number': '1',
},
method: 'POST',
duplex: 'half',
signal: internalAbortSignal,
},
);
Expand All @@ -252,7 +250,6 @@ describe('client', () => {
'x-mpu-part-number': '2',
},
method: 'POST',
duplex: 'half',
signal: internalAbortSignal,
},
);
Expand Down Expand Up @@ -376,7 +373,6 @@ describe('client', () => {
'x-mpu-part-number': '1',
},
method: 'POST',
duplex: 'half',
signal: internalAbortSignal,
},
);
Expand All @@ -396,7 +392,6 @@ describe('client', () => {
'x-mpu-part-number': '2',
},
method: 'POST',
duplex: 'half',
signal: internalAbortSignal,
},
);
Expand Down
11 changes: 7 additions & 4 deletions packages/blob/src/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type { IncomingMessage } from 'node:http';
// the `undici` module will be replaced with https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API
// for browser contexts. See ./undici-browser.js and ./package.json
import { fetch } from 'undici';
import type { BlobCommandOptions } from './helpers';
import type { BlobCommandOptions, WithUploadProgress } from './helpers';
import { BlobError, getTokenFromOptionsOrEnv } from './helpers';
import { createPutMethod } from './put';
import type { PutBlobResult } from './put-helpers';
Expand Down Expand Up @@ -42,7 +42,9 @@ export interface ClientTokenOptions {
}

// shared interface for put and upload
interface ClientCommonPutOptions extends ClientCommonCreateBlobOptions {
interface ClientCommonPutOptions
extends ClientCommonCreateBlobOptions,
WithUploadProgress {
/**
* Whether to use multipart upload. Use this when uploading large files. It will split the file into multiple parts, upload them in parallel and retry failed parts.
*/
Expand Down Expand Up @@ -89,7 +91,7 @@ export const put = createPutMethod<ClientPutCommandOptions>({
// vercelBlob. createMultipartUpload()
// vercelBlob. uploadPart()
// vercelBlob. completeMultipartUpload()
// vercelBlob. createMultipartUploaded()
// vercelBlob. createMultipartUploader()

export type ClientCreateMultipartUploadCommandOptions =
ClientCommonCreateBlobOptions & ClientTokenOptions;
Expand All @@ -110,7 +112,8 @@ export const createMultipartUploader =

type ClientMultipartUploadCommandOptions = ClientCommonCreateBlobOptions &
ClientTokenOptions &
CommonMultipartUploadOptions;
CommonMultipartUploadOptions &
WithUploadProgress;

export const uploadPart =
createUploadPartMethod<ClientMultipartUploadCommandOptions>({
Expand Down
Loading

0 comments on commit c3afec3

Please sign in to comment.