Use exponential backoff for s3 503 error

action/__tests__/plugins/staticHosting/helpers.test.ts

@@ -0,0 +1,47 @@
+import * as core from '@actions/core';
+import { afterEach, beforeEach, describe, expect, it, jest } from '@jest/globals';
+import fetch, { Response } from 'cross-fetch';
+
+import { uploadFileWithSignedUrl } from '../../../src/plugins/staticHosting/helpers';
+
+jest.mock('cross-fetch', () => jest.fn());
+const mockFetch = fetch as jest.MockedFunction<typeof fetch>;
+
+jest.mock('@actions/core');
+const mockCore = core as jest.Mocked<typeof core>;
+
+describe('uploadFileWithSignedUrl', function () {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  describe('when there is no error', function () {
+    it('uploads the file with one fetch call', async () => {
+      mockFetch.mockResolvedValue({ ok: true } as Response);
+      await uploadFileWithSignedUrl('https://localhost', {}, 'object-key', 'package.json');
+      expect(mockFetch).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('when there is 503 error', function () {
+    const response503 = { ok: false, status: 503, statusText: 'Slow down' } as Response;
+
+    beforeEach(() => {
+      mockFetch
+        .mockResolvedValueOnce(response503)
+        .mockResolvedValueOnce(response503)
+        .mockResolvedValue({ ok: true, status: 204, statusText: 'No content' } as Response);
+    });
+
+    it('retries', async () => {
+      await uploadFileWithSignedUrl('https://localhost', {}, 'object-key', 'package.json');
+      expect(mockFetch).toHaveBeenCalledTimes(3);
+
+      const warnings = mockCore.warning.mock.calls;
+      expect(warnings.length).toEqual(2);
+      warnings.forEach((warning) => {
+        expect(warning[0]).toContain('503 error');
+      });
+    });
+  });
+});

action/dist/index.js

@@ -28530,11 +28530,28 @@ const uploadFileWithSignedUrl = (signedUrl, fields, objectKey, localFilePath, dr
     form.append('key', objectKey);
     form.append('Content-Type', mime_types.lookup(localFilePath) || 'application/octet-stream');
     form.append('file', external_fs_default().readFileSync(localFilePath));
-    const response = yield node_ponyfill_default()(signedUrl, {
-        method: 'POST',
-        body: form
-    });
-    core.info(`-- Upload ${localFilePath} -> ${objectKey}: ${response.status} - ${response.statusText}`);
+    let retry = 0;
+    const maxRetry = 6;
+    while (retry <= maxRetry) {
+        const { ok, status, statusText } = yield node_ponyfill_default()(signedUrl, {
+            method: 'POST',
+            body: form
+        });
+        const retryStatus = retry > 0 ? ` (retry ${retry})` : '';
+        core.info(`-- Upload ${localFilePath} -> ${objectKey}: ${status} - ${statusText}${retryStatus}`);
+        if (ok) {
+            break;
+        }
+        else if (status === 503) {
+            const waitingMillis = Math.pow(2, retry) * 100;
+            core.warning(`-- Hit 503 error, waiting for ${waitingMillis}ms before retry (${retry})...`);
+            yield new Promise((r) => setTimeout(r, waitingMillis));
+        }
+        else {
+            throw new Error(statusText);
+        }
+        ++retry;
+    }
 });
 // Reference:
 // https://github.com/elysiumphase/s3-lambo/blob/master/lib/index.js#L255

action/src/plugins/staticHosting/helpers.ts

@@ -49,12 +49,27 @@ export const uploadFileWithSignedUrl = async (
   form.append('Content-Type', mime.lookup(localFilePath) || 'application/octet-stream');
   form.append('file', fs.readFileSync(localFilePath));
 
-  const response = await fetch(signedUrl, {
-    method: 'POST',
-    body: form as any
-  });
+  let retry = 0;
+  const maxRetry = 6;
+  while (retry <= maxRetry) {
+    const { ok, status, statusText } = await fetch(signedUrl, {
+      method: 'POST',
+      body: form as any
+    });
+    const retryStatus = retry > 0 ? ` (retry ${retry})` : '';
+    core.info(`-- Upload ${localFilePath} -> ${objectKey}: ${status} - ${statusText}${retryStatus}`);
 
-  core.info(`-- Upload ${localFilePath} -> ${objectKey}: ${response.status} - ${response.statusText}`);
+    if (ok) {
+      break;
+    } else if (status === 503) {
+      const waitingMillis = 2 ** retry * 100;
+      core.warning(`-- Hit 503 error, waiting for ${waitingMillis}ms before retry (${retry})...`);
+      await new Promise((r) => setTimeout(r, waitingMillis));
+    } else {
+      throw new Error(statusText);
+    }
+    ++retry;
+  }
 };
 
 // Reference: