feat: support for providing s3 endpoint url

README.md

@@ -110,6 +110,7 @@ Options:
   --shaka-executable [shakaExecutable]                Path to shaka-packager executable, defaults to 'packager'. Can also be set with environment variable SHAKA_PACKAGER_EXECUTABLE.
   --no-implicit-audio [noImplicitAudio]               Do not include audio unless audio input specified
   -d, --destination-folder <dest>                     Destination folder URL (supported protocols: s3, local file). Defaults to CWD.
+  --endpoint-url [s3EndpointUrl]                      S3 endpoint URL
   --dash-only                                         Package only DASH format
   --hls-only                                          Package only HLS format
   --segment-single-file                               Use byte range addressing and a single segment file per stream

src/cli.ts

@@ -54,6 +54,7 @@ cli
     '-d, --destination-folder <dest>',
     'Destination folder URL (supported protocols: s3, local file). Defaults to CWD.'
   )
+  .option('--endpoint-url [s3EndpointUrl]', 'S3 endpoint URL')
   .option('--dash-only', 'Package only DASH format')
   .option('--hls-only', 'Package only HLS format')
   .option(
@@ -94,6 +95,7 @@ cli
       );
       await doPackage({
         dest: options.destinationFolder || '.',
+        s3EndpointUrl: options.s3EndpointUrl,
         source: options.sourceFolder,
         inputs: inputOptions,
         stagingDir: options.stagingDir,

src/packager.ts

@@ -2,7 +2,7 @@ import path, { join } from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { existsSync, mkdirSync, rmSync, unlinkSync } from 'node:fs';
 import { readdir, mkdir } from 'node:fs/promises';
-import { toUrl, toUrlOrUndefined } from './util';
+import { createS3cmdArgs, toUrl, toUrlOrUndefined } from './util';
 import mv from 'mv';
 
 const DEFAULT_STAGING_DIR = '/tmp/data';
@@ -32,9 +32,10 @@ export interface PackageOptions {
   packageFormatOptions?: PackageFormatOptions;
   shakaExecutable?: string;
   serviceAccessToken?: string;
+  s3EndpointUrl?: string;
 }
 
-function validateOptios(opts: PackageOptions) {
+function validateOptions(opts: PackageOptions) {
   if (
     opts?.packageFormatOptions?.hlsOnly &&
     opts?.packageFormatOptions?.dashOnly
@@ -51,14 +52,14 @@ function validateOptios(opts: PackageOptions) {
 }
 
 export async function doPackage(opts: PackageOptions) {
-  validateOptios(opts);
+  validateOptions(opts);
   const stagingDir = await prepare(opts.stagingDir);
   await createPackage({ ...opts, stagingDir });
   if (toUrl(opts.dest).protocol === 's3:') {
     // We don't want to upload source files to S3
     await removeDownloadedFiles(opts.inputs, stagingDir);
   }
-  await uploadPackage(toUrl(opts.dest), stagingDir);
+  await uploadPackage(toUrl(opts.dest), stagingDir, opts.s3EndpointUrl);
   await cleanup(stagingDir);
 }
 
@@ -82,7 +83,8 @@ export async function download(
   input: Input,
   source?: URL,
   stagingDir?: string,
-  serviceAccessToken?: string
+  serviceAccessToken?: string,
+  endpointUrl?: string
 ): Promise<string> {
   if (!source) {
     return input.filename;
@@ -97,12 +99,11 @@ export async function download(
   if (source.protocol === 's3:') {
     const sourceFile = new URL(join(source.pathname, input.filename), source);
     const localFilename = join(stagingDir, input.filename);
-    const { status, stderr } = spawnSync('aws', [
-      's3',
-      'cp',
-      sourceFile.toString(),
-      localFilename
-    ]);
+    const args = createS3cmdArgs(
+      ['cp', sourceFile.toString(), localFilename],
+      endpointUrl
+    );
+    const { status, stderr } = spawnSync('aws', args);
     if (status !== 0) {
       if (stderr) {
         console.log(stderr.toString());
@@ -161,7 +162,11 @@ async function moveFile(src: string, dest: string) {
   });
 }
 
-export async function uploadPackage(dest: URL, stagingDir: string) {
+export async function uploadPackage(
+  dest: URL,
+  stagingDir: string,
+  s3EndpointUrl?: string
+) {
   if (!dest.protocol || dest.protocol === 'file:') {
     await mkdir(dest.pathname, { recursive: true });
     const files = await readdir(stagingDir);
@@ -174,13 +179,11 @@ export async function uploadPackage(dest: URL, stagingDir: string) {
   }
   if (dest.protocol === 's3:') {
     console.log(`Uploading package to ${dest.toString()}`);
-    const { status, stderr, error } = spawnSync('aws', [
-      's3',
-      'cp',
-      '--recursive',
-      stagingDir,
-      dest.toString()
-    ]);
+    const args = createS3cmdArgs(
+      ['cp', '--recursive', stagingDir, dest.toString()],
+      s3EndpointUrl
+    );
+    const { status, stderr, error } = spawnSync('aws', args);
     if (status !== 0) {
       if (error) {
         console.error(`Upload failed: ${error.message}`);
@@ -197,16 +200,23 @@ export async function uploadPackage(dest: URL, stagingDir: string) {
 }
 
 export async function createPackage(opts: PackageOptions) {
-  const { inputs, source, stagingDir, noImplicitAudio, serviceAccessToken } =
-    opts;
+  const {
+    inputs,
+    source,
+    stagingDir,
+    noImplicitAudio,
+    serviceAccessToken,
+    s3EndpointUrl
+  } = opts;
   const sourceUrl = toUrlOrUndefined(source);
   const downloadedInputs: Input[] = await Promise.all(
     inputs.map(async (input) => {
       const filename = await download(
         input,
         sourceUrl,
         stagingDir,
-        serviceAccessToken
+        serviceAccessToken,
+        s3EndpointUrl
       );
       return {
         ...input,

src/util.test.ts

@@ -0,0 +1,13 @@
+import { createS3cmdArgs } from './util';
+
+describe('cmd args helper', () => {
+  test('can construct s3cmd args without s3 endpoint url', () => {
+    const args = createS3cmdArgs(['ls']);
+    expect(args).toEqual(['s3', 'ls']);
+  });
+
+  test('can construct s3cmd args s3 endpoint url', () => {
+    const args = createS3cmdArgs(['ls'], 'https://my.api.url.com');
+    expect(args).toEqual(['s3', '--endpoint-url=https://my.api.url.com', 'ls']);
+  });
+});

src/util.ts

@@ -12,3 +12,14 @@ export function toUrl(url: string) {
     ? new URL(url)
     : new URL(`file://${path.resolve(url)}`);
 }
+
+export function createS3cmdArgs(
+  cmdArgs: string[],
+  s3EndpointUrl?: string
+): string[] {
+  const args = ['s3'];
+  if (s3EndpointUrl) {
+    args.push(`--endpoint-url=${s3EndpointUrl}`);
+  }
+  return args.concat(cmdArgs);
+}