Skip to content

Commit

Permalink
Versionize docker image & add e2e (#33)
Browse files Browse the repository at this point in the history
  • Loading branch information
Leslie-Wong-H authored Apr 14, 2023
2 parents 9892ea8 + d981de6 commit c6f67fa
Show file tree
Hide file tree
Showing 10 changed files with 215 additions and 136 deletions.
1 change: 0 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ TRACE_API_SECRET=TRACE_API_SECRET
IP_WHITELIST=192.168.1.100
AWS_BUCKET=
AWS_ENDPOINT_URL=
# AWS_HLS_URL=
AWS_REGION=
AWS_ACCESS_KEY=
AWS_SECRET_KEY=
19 changes: 11 additions & 8 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ name: Docker Image CI
on:
push:
branches: [main]
tags: [v*]
pull_request:
branches: [main]

jobs:
build:
docker-image-build:
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest

steps:
- name: Check Out Repo
uses: actions/checkout@v2
Expand All @@ -22,7 +25,7 @@ jobs:
run: |
echo "OWNER_LC=${OWNER,,}" >> ${GITHUB_ENV}
env:
OWNER: '${{ github.repository_owner }}'
OWNER: "${{ github.repository_owner }}"

- name: Login to GitHub Container Registry
uses: docker/login-action@v1
Expand All @@ -39,14 +42,14 @@ jobs:
run: |
echo "REPOSITORY_LC=${REPOSITORY,,}" >> ${GITHUB_ENV}
env:
REPOSITORY: '${{ github.repository }}'
REPOSITORY: "${{ github.repository }}"

- name: Set correct DockerHub repository name
run: |
echo "DOCKERHUB_REPOSITORY_LC=${DOCKERHUB_USERNAME,,}/${REPOSITORY#*/}" >> ${GITHUB_ENV}
env:
DOCKERHUB_USERNAME: '${{ secrets.DOCKER_HUB_USERNAME}}'
REPOSITORY: '${{ github.repository }}'
DOCKERHUB_USERNAME: "${{ secrets.DOCKER_HUB_USERNAME}}"
REPOSITORY: "${{ github.repository }}"

- name: Build and push
id: docker_build
Expand All @@ -56,8 +59,8 @@ jobs:
file: ./Dockerfile
push: true
tags: |
${{ env.DOCKERHUB_REPOSITORY_LC }}:latest
ghcr.io/${{ env.REPOSITORY_LC }}:latest
${{ env.DOCKERHUB_REPOSITORY_LC }}:${{ github.ref_name }}
ghcr.io/${{ env.REPOSITORY_LC }}:${{ github.ref_name }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
24 changes: 22 additions & 2 deletions .github/workflows/node.js.yml → .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions

name: Node.js CI
name: e2e Node.js CI

on:
push:
Expand All @@ -24,6 +24,26 @@ jobs:
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: Install ffmpeg
run: |
sudo apt-get update -y
sudo apt-get install -y ffmpeg --fix-missing
- run: yarn install --frozen-lockfile
# - run: yarn build --if-present
- run: yarn test
- run: yarn jest
env:
SERVER_PORT: 8024
SERVER_ADDR: 127.0.0.1
TRACE_MEDIA_SALT: TRACE_MEDIA_SALT
TRACE_API_SECRET: TRACE_API_SECRET
IP_WHITELIST: ${{ secrets.IP_WHITELIST }}
AWS_BUCKET: ${{ secrets.AWS_BUCKET }}
AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
- name: Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: unittests
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
"sanitize-filename": "^1.6.3"
},
"devDependencies": {
"abort-controller": "^3.0.0",
"cross-env": "^7.0.3",
"jest": "^29.5.0",
"prettier": "2.8.0",
Expand Down
3 changes: 2 additions & 1 deletion src/file.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import http from "http";
import fs from "fs-extra";
import { default as request } from "supertest";
import fetch, { AbortError } from "node-fetch";
import abortController from "abort-controller";
import app from "./app.js";

const { TRACE_API_SECRET } = process.env;
Expand Down Expand Up @@ -137,7 +138,7 @@ test(
const filename = "Big Buck Bunny.mp4";
const fileBuffer = fs.createReadStream(videoFilePath);
// // AbortController was added in node v14.17.0 globally
const AbortController = globalThis.AbortController || (await import("abort-controller"));
const AbortController = globalThis.AbortController || abortController;
const controller = new AbortController();
const timeout = setTimeout(() => {
controller.abort();
Expand Down
19 changes: 0 additions & 19 deletions src/image.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import child_process from "child_process";

const {
AWS_ENDPOINT_URL,
// AWS_HLS_URL,
AWS_ACCESS_KEY,
AWS_SECRET_KEY,
AWS_BUCKET,
Expand Down Expand Up @@ -105,30 +104,12 @@ export default async (req, res) => {
return res.status(400).send("Bad Request. Invalid param: size");
}
try {
///////////////////////////
// mp4 version: //
// (Note: now handing hls//
// files because of the //
// param 'hls' key above //
// and it works.) :) //
///////////////////////////

command = new GetObjectCommand(params);
const signedUrl = await getSignedUrl(s3, command, { expiresIn: 60 * 5 });

const image = generateImagePreview(signedUrl, t, size);
res.set("Content-Type", "image/jpg");
res.send(image);

//////////////////////////
// HLS version: //
//////////////////////////

// // Note: AWS S3 prefix authentication and CORS config
// const targetHlsUrl = `${AWS_HLS_URL}/${params.Key}`;
// const image = generateImagePreview(targetHlsUrl, t, size);
// res.set("Content-Type", "image/jpg");
// res.send(image);
} catch (e) {
console.log(e);
res.status(500).send("Internal Server Error");
Expand Down
146 changes: 146 additions & 0 deletions src/lib/generate-video-preview.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import fs from "fs-extra";
import path from "path";
import os from "os";
import crypto from "crypto";
import fetch from "node-fetch";
import child_process from "child_process";
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";

const { AWS_ENDPOINT_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_BUCKET, AWS_REGION } = process.env;

const opts = AWS_ENDPOINT_URL
? {
forcePathStyle: true,
endpoint: AWS_ENDPOINT_URL,
region: AWS_REGION,
credentials: {
accessKeyId: AWS_ACCESS_KEY,
secretAccessKey: AWS_SECRET_KEY,
},
}
: {};

const s3 = new S3Client(opts);

let command;

export default async (filePath, start, end, key, size = "m", mute = false) => {
const tempPath = path.join(os.tmpdir(), crypto.createHash("md5").update(filePath).digest("hex"));
fs.ensureDirSync(tempPath);

const tempIndexPath = path.join(tempPath, "index.m3u8");

const response = await fetch(filePath);
const downloadNecessaryHLS = (res, path) => {
return new Promise((resolve) => {
res.body.pipe(fs.createWriteStream(path));
res.body.on("end", async () => {
console.log(`Fetched ${path}`);
const cont = await fs.readFile(path, { encoding: "utf8" });
const lines = cont.split("\n");
let tsList = [];
lines.reduce((acc, curV, curI) => {
let re = /^#EXTINF:(?<num>\d+\.?\d*),/;
let match = re.exec(curV);
let sum = acc;
if (match) {
const {
groups: { num },
} = match;
sum = acc + Number(num);
if (start >= acc && start <= sum) {
// allow over edge
tsList.push(`10s_${(curI - 2 + "").padStart(3, "0")}.ts`);
tsList.push(`10s_${(curI - 1 + "").padStart(3, "0")}.ts`);
tsList.push(`10s_${(curI + "").padStart(3, "0")}.ts`);
tsList.push(`10s_${(curI + 1 + "").padStart(3, "0")}.ts`);
tsList.push(`10s_${(curI + 2 + "").padStart(3, "0")}.ts`);
}
}
return sum;
}, 0);
let signedUrl = "";
let params;
for (const ts of tsList) {
try {
params = {
Bucket: AWS_BUCKET,
Key: `${key}/${ts}`,
};
command = new GetObjectCommand(params);
signedUrl = await getSignedUrl(s3, command, { expiresIn: 60 * 5 });
const tsResponse = await fetch(signedUrl);
fs.writeFileSync(path.join(tempPath, ts), tsResponse);
} catch (error) {
console.log(error);
}
}
await resolve("ok");
});
});
};

await downloadNecessaryHLS(response, tempIndexPath);

const ffmpeg = child_process.spawnSync(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-nostats",
"-headers",
"Referer: https://shotit.github.io/",
"-y",
"-ss",
start - 10,
"-i",
tempIndexPath,
"-ss",
"10",
"-t",
end - start,
mute ? "-an" : "-y",
"-map",
"0:v:0",
"-map",
"0:a:0",
"-vf",
`scale=${{ l: 640, m: 320, s: 160 }[size]}:-2`,
"-c:v",
"libx264",
"-crf",
"23",
"-profile:v",
"high",
"-preset",
"faster",
"-r",
"24000/1001",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
"-b:a",
"128k",
"-max_muxing_queue_size",
"1024",
"-movflags",
"empty_moov",
"-map_metadata",
"-1",
"-map_chapters",
"-1",
"-f",
"mp4",
"-",
],
{ maxBuffer: 1024 * 1024 * 100 }
);
if (ffmpeg.stderr.length) {
console.log(ffmpeg.stderr.toString());
}
fs.rmdirSync(tempPath, { recursive: true, force: true });
return ffmpeg.stdout;
};
14 changes: 9 additions & 5 deletions src/list.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,15 @@ import app from "./app.js";

const { TRACE_API_SECRET } = process.env;

test("GET /list correct", async () => {
const response = await request(app).get("/list").set("x-trace-secret", TRACE_API_SECRET);
expect(response.statusCode).toBe(200);
expect(response.body[0].Key).toBeDefined();
});
test(
"GET /list correct",
async () => {
const response = await request(app).get("/list").set("x-trace-secret", TRACE_API_SECRET);
expect(response.statusCode).toBe(200);
expect(response.body[0].Key).toBeDefined();
},
30 * 1000
);

test("GET /list/tt1254207", async () => {
const response = await request(app)
Expand Down
Loading

0 comments on commit c6f67fa

Please sign in to comment.